blob: c30af6b746a7be579cc3c19ddbc513617265d2a8 [file] [log] [blame]
#!/usr/bin/env dart
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:args/args.dart';
import 'package:glob/glob.dart';
final parser = ArgParser()
..addMultiOption('bot',
abbr: 'b',
help: 'Select the bots matching the glob pattern [option is repeatable]',
splitCommas: false)
..addFlag('verbose', abbr: 'v', help: 'Verbose output.', negatable: false)
..addFlag('help', help: 'Show the program usage.', negatable: false);
void printUsage() {
print("""
Usage: ${Platform.executable} ${Platform.script} [OLDER_COMMIT] [NEWER_COMMIT]
The options are as follows:
${parser.usage}""");
}
late bool verbose;
void main(List<String> args) async {
final options = parser.parse(args);
if (options.flag("help")) {
printUsage();
return;
}
final commits = options.rest;
if (commits.length < 2) {
print('Need to supply at least two commits.');
printUsage();
exitCode = 1;
return;
}
verbose = options.flag('verbose');
final globs = List<Glob>.from(
options.multiOption('bot').map((pattern) => Glob(pattern)));
final vmBuilders = loadVmBuildersFromTestMatrix(globs);
final futures = <Future<List<Result>>>[];
for (final commit in commits) {
final DateTime date = await getDateOfCommit(commit);
futures.add(getResults(commit, date, vmBuilders));
}
final results = await Future.wait(futures);
for (int i = 0; i < results.length - 1; i++) {
final commitB = commits[i];
final commitA = commits[i + 1];
print('\nResult changes between $commitB -> $commitA:');
final commonGroups =
buildCommonGroups(commitA, commitB, results[i], results[i + 1]);
for (final commonGroup in commonGroups) {
final builders = commonGroup.builders;
print('');
for (final group in commonGroup.groups) {
final diff = group.diffs.first;
print('${group.test} ${diff.before} -> ${diff.after}');
}
for (final b in extractBuilderPattern(builders)) {
print(' on $b');
}
}
}
}
Future<DateTime> getDateOfCommit(String commit) async {
final result = await Process.run(
'git', ['show', '-s', '--format=%cd', '--date=iso-strict', commit]);
if (result.exitCode != 0) {
print('Could not determine date of commit $commit. Git reported:\n');
print(result.stdout);
print(result.stderr);
exit(1);
}
return DateTime.parse((result.stdout as String).trim());
}
Future<List<Result>> getResults(
String commit, DateTime dateC, Set<String> builders) async {
final DateTime date0 = dateC.add(const Duration(hours: 24));
final DateTime date2 = dateC.subtract(const Duration(hours: 24));
final query = '''
SELECT commit_time, builder_name, build_number, name, result, expected FROM `dart-ci.results.results`
WHERE commit_hash="$commit"
AND matches=false
AND (_PARTITIONDATE = "${formatDate(date0)}" OR
_PARTITIONDATE = "${formatDate(dateC)}" OR
_PARTITIONDATE = "${formatDate(date2)}" )
AND (STARTS_WITH(builder_name, "vm-") OR
STARTS_WITH(builder_name, "app-") OR
STARTS_WITH(builder_name, "cross-"))
AND ((flaky is NULL) OR flaky=false)
ORDER BY name''';
final arguments = <String>[
'query',
'--format=prettyjson',
'--project_id=dart-ci',
'--nouse_legacy_sql',
'-n',
'1000000',
query,
];
if (verbose) {
print('Executing query:\n bq ${arguments.join(' ')}');
}
final result = await Process.run('bq', arguments);
if (result.exitCode == 0) {
File('$commit.json').writeAsStringSync(result.stdout);
final resultsForCommit = json.decode(result.stdout);
final results = <Result>[];
for (final Map<String, dynamic> result in resultsForCommit) {
final builderName = result['builder_name'];
if (!builders.contains(builderName)) {
continue;
}
final failure = Result(commit, builderName, result['build_number'],
result['name'], result['expected'], result['result']);
results.add(failure);
}
results.sort((Result a, Result b) {
final c = a.name.compareTo(b.name);
if (c != 0) return c;
return a.builderName.compareTo(b.builderName);
});
return results;
} else {
print('Running the following query failed:\nbq ${arguments.join(' ')}');
print('Exit code: ${result.exitCode}');
final stdout = (result.stdout as String).trim();
if (stdout.isNotEmpty) {
print('Stdout:\n$stdout');
}
final stderr = (result.stderr as String).trim();
if (stderr.isNotEmpty) {
print('Stderr:\n$stderr');
}
return <Result>[];
}
}
List<CommonGroup> buildCommonGroups(String commitA, String commitB,
List<Result> commitResults, List<Result> commitResultsBefore) {
// If a test has same outcome across many vm builders
final diffs = <Diff>[];
int i = 0;
int j = 0;
while (i < commitResultsBefore.length && j < commitResults.length) {
final a = commitResultsBefore[i];
final b = commitResults[j];
// Is a smaller than b, then we had a failure before and no longer one.
if (a.name.compareTo(b.name) < 0 ||
(a.name.compareTo(b.name) == 0 &&
a.builderName.compareTo(b.builderName) < 0)) {
diffs.add(Diff(a, null));
i++;
continue;
}
// Is b smaller than a, then we had no failure before but have one now.
if (b.name.compareTo(a.name) < 0 ||
(b.name.compareTo(a.name) == 0 &&
b.builderName.compareTo(a.builderName) < 0)) {
diffs.add(Diff(null, b));
j++;
continue;
}
// Else we must have the same name and builder.
if (a.name != b.name || a.builderName != b.builderName) throw 'BUG';
if (a.expected != b.expected || a.result != b.result) {
diffs.add(Diff(a, b));
}
i++;
j++;
}
while (i < commitResultsBefore.length) {
final a = commitResultsBefore[i++];
diffs.add(Diff(a, null));
}
while (j < commitResults.length) {
final b = commitResults[j++];
diffs.add(Diff(null, b));
}
// If a test has same outcome across many vm builders
final groups = <GroupedDiff>[];
int h = 0;
while (h < diffs.length) {
final d = diffs[h++];
final builders = <String>{}..add(d.builder);
final groupDiffs = <Diff>[d];
while (h < diffs.length) {
final nd = diffs[h];
if (d.test == nd.test) {
if (d.sameExpectationDifferenceAs(nd)) {
builders.add(nd.builder);
groupDiffs.add(nd);
h++;
continue;
}
}
break;
}
groups.add(GroupedDiff(d.test, builders.toList()..sort(), groupDiffs));
}
final commonGroups = <String, List<GroupedDiff>>{};
for (final group in groups) {
final key = group.builders.join(' ');
commonGroups.putIfAbsent(key, () => <GroupedDiff>[]).add(group);
}
final commonGroupList = commonGroups.values
.map((list) => CommonGroup(list.first.builders, list))
.toList();
commonGroupList
.sort((a, b) => a.builders.length.compareTo(b.builders.length));
return commonGroupList;
}
class CommonGroup {
final List<String> builders;
final List<GroupedDiff> groups;
CommonGroup(this.builders, this.groups);
}
class GroupedDiff {
final String test;
final List<String> builders;
final List<Diff> diffs;
GroupedDiff(this.test, this.builders, this.diffs);
}
class Diff {
final Result? before;
final Result? after;
Diff(this.before, this.after);
String get test => (before?.name ?? after?.name)!;
String get builder => (before?.builderName ?? after?.builderName)!;
bool sameExpectationDifferenceAs(Diff other) {
if ((before == null) != (other.before == null)) return false;
if ((after == null) != (other.after == null)) return false;
if (before != null) {
if (!before!.sameResult(other.before!)) return false;
}
if (after != null) {
if (!after!.sameResult(other.after!)) return false;
}
return true;
}
}
class Result {
final String commit;
final String builderName;
final String buildNumber;
final String name;
final String expected;
final String result;
Result(this.commit, this.builderName, this.buildNumber, this.name,
this.expected, this.result);
@override
String toString() => '(expected: $expected, actual: $result)';
bool sameResult(Result other) {
return name == other.name &&
expected == other.expected &&
result == other.result;
}
bool equals(Object other) {
if (other is Result) {
if (name != other.name) return false;
if (builderName != other.builderName) return false;
}
return false;
}
@override
int get hashCode => name.hashCode ^ builderName.hashCode;
@override
bool operator ==(Object other) {
// TODO: implement ==
return super == other;
}
}
String currentDate() {
final timestamp = DateTime.now().toUtc().toIso8601String();
return timestamp.substring(0, timestamp.indexOf('T'));
}
Set<String> loadVmBuildersFromTestMatrix(List<Glob> globs) {
final contents = File('tools/bots/test_matrix.json').readAsStringSync();
final testMatrix = json.decode(contents) as Map<String, dynamic>;
final vmBuilders = <String>{};
for (final config in testMatrix['builder_configurations']) {
for (final builder in (config as Map)['builders']) {
// Cast to a string.
builder as String;
if (builder.startsWith('vm-') || builder.startsWith('app-')) {
vmBuilders.add(builder);
}
}
}
// This one is in the test_matrix.json but we don't run it on CI.
vmBuilders.remove('vm-kernel-asan-linux-release-ia32');
if (globs.isNotEmpty) {
vmBuilders.removeWhere((String builder) {
return !globs.any((Glob glob) => glob.matches(builder));
});
}
return vmBuilders;
}
List<String> extractBuilderPattern(List<String> builders) {
final all = Set<String>.from(builders);
String reduce(String builder, List<String> possibilities) {
for (final pos in possibilities) {
if (builder.contains(pos)) {
final existing = <String>[];
final available = <String>[];
for (final pos2 in possibilities) {
final builder2 = builder.replaceFirst(pos, pos2);
if (all.contains(builder2)) {
existing.add(builder2);
available.add(pos2);
}
}
if (existing.length > 1) {
all.removeAll(existing);
final replacement =
builder.replaceFirst(pos, '{${available.join(',')}}');
all.add(replacement);
return replacement;
}
}
}
return builder;
}
for (String builder in builders) {
if (all.contains(builder)) {
builder = reduce(builder, const ['debug', 'release', 'product']);
}
}
for (String builder in all.toList()) {
if (all.contains(builder)) {
builder = reduce(builder, const ['mac', 'linux', 'win']);
}
}
for (String builder in all.toList()) {
if (all.contains(builder)) {
builder = reduce(builder, const [
'ia32',
'x64',
'simarm',
'simarm64',
'arm',
'arm64',
]);
}
}
return all.toList()..sort();
}
String formatDate(DateTime date) {
final s = date.toIso8601String();
return s.substring(0, s.indexOf('T'));
}