| #!/usr/bin/env dart |
| // Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'dart:async'; |
| import 'dart:convert'; |
| import 'dart:io'; |
| |
| import 'package:args/args.dart'; |
| import 'package:glob/glob.dart'; |
| |
| final parser = ArgParser() |
| ..addMultiOption('bot', |
| abbr: 'b', |
| help: 'Select the bots matching the glob pattern [option is repeatable]', |
| splitCommas: false) |
| ..addFlag('verbose', abbr: 'v', help: 'Verbose output.', negatable: false) |
| ..addFlag('help', help: 'Show the program usage.', negatable: false); |
| |
| void printUsage() { |
| print(""" |
| Usage: ${Platform.executable} ${Platform.script} [OLDER_COMMIT] [NEWER_COMMIT] |
| |
| The options are as follows: |
| |
| ${parser.usage}"""); |
| } |
| |
| late bool verbose; |
| |
| void main(List<String> args) async { |
| final options = parser.parse(args); |
| if (options.flag("help")) { |
| printUsage(); |
| return; |
| } |
| |
| final commits = options.rest; |
| if (commits.length < 2) { |
| print('Need to supply at least two commits.'); |
| printUsage(); |
| exitCode = 1; |
| return; |
| } |
| verbose = options.flag('verbose'); |
| |
| final globs = List<Glob>.from( |
| options.multiOption('bot').map((pattern) => Glob(pattern))); |
| final vmBuilders = loadVmBuildersFromTestMatrix(globs); |
| |
| final futures = <Future<List<Result>>>[]; |
| for (final commit in commits) { |
| final DateTime date = await getDateOfCommit(commit); |
| futures.add(getResults(commit, date, vmBuilders)); |
| } |
| |
| final results = await Future.wait(futures); |
| for (int i = 0; i < results.length - 1; i++) { |
| final commitB = commits[i]; |
| final commitA = commits[i + 1]; |
| |
| print('\nResult changes between $commitB -> $commitA:'); |
| final commonGroups = |
| buildCommonGroups(commitA, commitB, results[i], results[i + 1]); |
| for (final commonGroup in commonGroups) { |
| final builders = commonGroup.builders; |
| |
| print(''); |
| for (final group in commonGroup.groups) { |
| final diff = group.diffs.first; |
| print('${group.test} ${diff.before} -> ${diff.after}'); |
| } |
| for (final b in extractBuilderPattern(builders)) { |
| print(' on $b'); |
| } |
| } |
| } |
| } |
| |
| Future<DateTime> getDateOfCommit(String commit) async { |
| final result = await Process.run( |
| 'git', ['show', '-s', '--format=%cd', '--date=iso-strict', commit]); |
| if (result.exitCode != 0) { |
| print('Could not determine date of commit $commit. Git reported:\n'); |
| print(result.stdout); |
| print(result.stderr); |
| exit(1); |
| } |
| return DateTime.parse((result.stdout as String).trim()); |
| } |
| |
| Future<List<Result>> getResults( |
| String commit, DateTime dateC, Set<String> builders) async { |
| final DateTime date0 = dateC.add(const Duration(hours: 24)); |
| final DateTime date2 = dateC.subtract(const Duration(hours: 24)); |
| final query = ''' |
| SELECT commit_time, builder_name, build_number, name, result, expected FROM `dart-ci.results.results` |
| WHERE commit_hash="$commit" |
| AND matches=false |
| AND (_PARTITIONDATE = "${formatDate(date0)}" OR |
| _PARTITIONDATE = "${formatDate(dateC)}" OR |
| _PARTITIONDATE = "${formatDate(date2)}" ) |
| AND (STARTS_WITH(builder_name, "vm-") OR |
| STARTS_WITH(builder_name, "app-") OR |
| STARTS_WITH(builder_name, "cross-")) |
| AND ((flaky is NULL) OR flaky=false) |
| ORDER BY name'''; |
| |
| final arguments = <String>[ |
| 'query', |
| '--format=prettyjson', |
| '--project_id=dart-ci', |
| '--nouse_legacy_sql', |
| '-n', |
| '1000000', |
| query, |
| ]; |
| if (verbose) { |
| print('Executing query:\n bq ${arguments.join(' ')}'); |
| } |
| |
| final result = await Process.run('bq', arguments); |
| if (result.exitCode == 0) { |
| File('$commit.json').writeAsStringSync(result.stdout); |
| final resultsForCommit = json.decode(result.stdout); |
| |
| final results = <Result>[]; |
| for (final Map<String, dynamic> result in resultsForCommit) { |
| final builderName = result['builder_name']; |
| if (!builders.contains(builderName)) { |
| continue; |
| } |
| |
| final failure = Result(commit, builderName, result['build_number'], |
| result['name'], result['expected'], result['result']); |
| results.add(failure); |
| } |
| |
| results.sort((Result a, Result b) { |
| final c = a.name.compareTo(b.name); |
| if (c != 0) return c; |
| return a.builderName.compareTo(b.builderName); |
| }); |
| |
| return results; |
| } else { |
| print('Running the following query failed:\nbq ${arguments.join(' ')}'); |
| print('Exit code: ${result.exitCode}'); |
| final stdout = (result.stdout as String).trim(); |
| if (stdout.isNotEmpty) { |
| print('Stdout:\n$stdout'); |
| } |
| final stderr = (result.stderr as String).trim(); |
| if (stderr.isNotEmpty) { |
| print('Stderr:\n$stderr'); |
| } |
| return <Result>[]; |
| } |
| } |
| |
| List<CommonGroup> buildCommonGroups(String commitA, String commitB, |
| List<Result> commitResults, List<Result> commitResultsBefore) { |
| // If a test has same outcome across many vm builders |
| final diffs = <Diff>[]; |
| int i = 0; |
| int j = 0; |
| while (i < commitResultsBefore.length && j < commitResults.length) { |
| final a = commitResultsBefore[i]; |
| final b = commitResults[j]; |
| |
| // Is a smaller than b, then we had a failure before and no longer one. |
| if (a.name.compareTo(b.name) < 0 || |
| (a.name.compareTo(b.name) == 0 && |
| a.builderName.compareTo(b.builderName) < 0)) { |
| diffs.add(Diff(a, null)); |
| i++; |
| continue; |
| } |
| |
| // Is b smaller than a, then we had no failure before but have one now. |
| if (b.name.compareTo(a.name) < 0 || |
| (b.name.compareTo(a.name) == 0 && |
| b.builderName.compareTo(a.builderName) < 0)) { |
| diffs.add(Diff(null, b)); |
| j++; |
| continue; |
| } |
| |
| // Else we must have the same name and builder. |
| if (a.name != b.name || a.builderName != b.builderName) throw 'BUG'; |
| |
| if (a.expected != b.expected || a.result != b.result) { |
| diffs.add(Diff(a, b)); |
| } |
| i++; |
| j++; |
| } |
| |
| while (i < commitResultsBefore.length) { |
| final a = commitResultsBefore[i++]; |
| diffs.add(Diff(a, null)); |
| } |
| |
| while (j < commitResults.length) { |
| final b = commitResults[j++]; |
| diffs.add(Diff(null, b)); |
| } |
| |
| // If a test has same outcome across many vm builders |
| final groups = <GroupedDiff>[]; |
| int h = 0; |
| while (h < diffs.length) { |
| final d = diffs[h++]; |
| final builders = <String>{}..add(d.builder); |
| final groupDiffs = <Diff>[d]; |
| |
| while (h < diffs.length) { |
| final nd = diffs[h]; |
| if (d.test == nd.test) { |
| if (d.sameExpectationDifferenceAs(nd)) { |
| builders.add(nd.builder); |
| groupDiffs.add(nd); |
| h++; |
| continue; |
| } |
| } |
| break; |
| } |
| |
| groups.add(GroupedDiff(d.test, builders.toList()..sort(), groupDiffs)); |
| } |
| |
| final commonGroups = <String, List<GroupedDiff>>{}; |
| for (final group in groups) { |
| final key = group.builders.join(' '); |
| commonGroups.putIfAbsent(key, () => <GroupedDiff>[]).add(group); |
| } |
| |
| final commonGroupList = commonGroups.values |
| .map((list) => CommonGroup(list.first.builders, list)) |
| .toList(); |
| commonGroupList |
| .sort((a, b) => a.builders.length.compareTo(b.builders.length)); |
| return commonGroupList; |
| } |
| |
| class CommonGroup { |
| final List<String> builders; |
| final List<GroupedDiff> groups; |
| CommonGroup(this.builders, this.groups); |
| } |
| |
| class GroupedDiff { |
| final String test; |
| final List<String> builders; |
| final List<Diff> diffs; |
| |
| GroupedDiff(this.test, this.builders, this.diffs); |
| } |
| |
| class Diff { |
| final Result? before; |
| final Result? after; |
| |
| Diff(this.before, this.after); |
| |
| String get test => (before?.name ?? after?.name)!; |
| String get builder => (before?.builderName ?? after?.builderName)!; |
| |
| bool sameExpectationDifferenceAs(Diff other) { |
| if ((before == null) != (other.before == null)) return false; |
| if ((after == null) != (other.after == null)) return false; |
| |
| if (before != null) { |
| if (!before!.sameResult(other.before!)) return false; |
| } |
| if (after != null) { |
| if (!after!.sameResult(other.after!)) return false; |
| } |
| return true; |
| } |
| } |
| |
| class Result { |
| final String commit; |
| final String builderName; |
| final String buildNumber; |
| final String name; |
| final String expected; |
| final String result; |
| |
| Result(this.commit, this.builderName, this.buildNumber, this.name, |
| this.expected, this.result); |
| |
| @override |
| String toString() => '(expected: $expected, actual: $result)'; |
| |
| bool sameResult(Result other) { |
| return name == other.name && |
| expected == other.expected && |
| result == other.result; |
| } |
| |
| bool equals(Object other) { |
| if (other is Result) { |
| if (name != other.name) return false; |
| if (builderName != other.builderName) return false; |
| } |
| return false; |
| } |
| |
| @override |
| int get hashCode => name.hashCode ^ builderName.hashCode; |
| |
| @override |
| bool operator ==(Object other) { |
| // TODO: implement == |
| return super == other; |
| } |
| } |
| |
| String currentDate() { |
| final timestamp = DateTime.now().toUtc().toIso8601String(); |
| return timestamp.substring(0, timestamp.indexOf('T')); |
| } |
| |
| Set<String> loadVmBuildersFromTestMatrix(List<Glob> globs) { |
| final contents = File('tools/bots/test_matrix.json').readAsStringSync(); |
| final testMatrix = json.decode(contents) as Map<String, dynamic>; |
| |
| final vmBuilders = <String>{}; |
| for (final config in testMatrix['builder_configurations']) { |
| for (final builder in (config as Map)['builders']) { |
| // Cast to a string. |
| builder as String; |
| |
| if (builder.startsWith('vm-') || builder.startsWith('app-')) { |
| vmBuilders.add(builder); |
| } |
| } |
| } |
| |
| // This one is in the test_matrix.json but we don't run it on CI. |
| vmBuilders.remove('vm-kernel-asan-linux-release-ia32'); |
| |
| if (globs.isNotEmpty) { |
| vmBuilders.removeWhere((String builder) { |
| return !globs.any((Glob glob) => glob.matches(builder)); |
| }); |
| } |
| |
| return vmBuilders; |
| } |
| |
| List<String> extractBuilderPattern(List<String> builders) { |
| final all = Set<String>.from(builders); |
| |
| String reduce(String builder, List<String> possibilities) { |
| for (final pos in possibilities) { |
| if (builder.contains(pos)) { |
| final existing = <String>[]; |
| final available = <String>[]; |
| for (final pos2 in possibilities) { |
| final builder2 = builder.replaceFirst(pos, pos2); |
| if (all.contains(builder2)) { |
| existing.add(builder2); |
| available.add(pos2); |
| } |
| } |
| if (existing.length > 1) { |
| all.removeAll(existing); |
| final replacement = |
| builder.replaceFirst(pos, '{${available.join(',')}}'); |
| all.add(replacement); |
| return replacement; |
| } |
| } |
| } |
| return builder; |
| } |
| |
| for (String builder in builders) { |
| if (all.contains(builder)) { |
| builder = reduce(builder, const ['debug', 'release', 'product']); |
| } |
| } |
| for (String builder in all.toList()) { |
| if (all.contains(builder)) { |
| builder = reduce(builder, const ['mac', 'linux', 'win']); |
| } |
| } |
| |
| for (String builder in all.toList()) { |
| if (all.contains(builder)) { |
| builder = reduce(builder, const [ |
| 'ia32', |
| 'x64', |
| 'simarm', |
| 'simarm64', |
| 'arm', |
| 'arm64', |
| ]); |
| } |
| } |
| return all.toList()..sort(); |
| } |
| |
| String formatDate(DateTime date) { |
| final s = date.toIso8601String(); |
| return s.substring(0, s.indexOf('T')); |
| } |