[baseline] Allow baselining a builder from multiple other builders
The script now takes a list of source builders (e.g. `--builders=a,b`).
The target builder is passed separately (e.g. `--target=c`). The results
from all source builders are mapped using the specified configuration
mappings and combined into the target builders results.
Bug: b/210809535
Change-Id: Ib23fff5d326977b06ef408ad29c21eefaf06f770
Reviewed-on: https://dart-review.googlesource.com/c/dart_ci/+/224201
Reviewed-by: William Hesse <whesse@google.com>
Commit-Queue: Alexander Thomas <athom@google.com>
diff --git a/baseline/lib/baseline.dart b/baseline/lib/baseline.dart
index b5eac65..6a93a70 100644
--- a/baseline/lib/baseline.dart
+++ b/baseline/lib/baseline.dart
@@ -6,6 +6,8 @@
import 'dart:convert';
import 'dart:io';
+import 'package:pool/pool.dart';
+
const _resultBase = 'gs://dart-test-results/builders';
/// Baselines a builder with the [options] and copies the results to the
@@ -17,43 +19,46 @@
if (channel != 'main') {
futures.add(baselineBuilder(
options.builders.map((b) => '$b-$channel').toList(),
+ '${options.target}-$channel',
options.configs,
options.dryRun,
resultBase));
} else {
- futures.add(baselineBuilder(
- options.builders, options.configs, options.dryRun, resultBase));
+ futures.add(baselineBuilder(options.builders, options.target,
+ options.configs, options.dryRun, resultBase));
}
}
await Future.wait(futures);
}
-Future<void> baselineBuilder(List<String> builders, Map<String, String> configs,
- bool dryRun, String resultBase) async {
- var from = builders[0];
- var to = builders[1];
- var latest = await read('$resultBase/$from/latest');
- var results = await read('$resultBase/$from/$latest/results.json');
+Future<void> baselineBuilder(List<String> builders, String target,
+ Map<String, String> configs, bool dryRun, String resultBase) async {
+ var resultsStream = Pool(4).forEach(builders, (builder) async {
+ var latest = await read('$resultBase/$builder/latest');
+ return await read('$resultBase/$builder/$latest/results.json');
+ });
var modifiedResults = StringBuffer();
- for (var json in LineSplitter.split(results).map(jsonDecode)) {
- json['build_number'] = 0;
- json['previous_build_number'] = 0;
- json['builder_name'] = to;
- var configuration = configs[json['configuration']];
- if (configuration == null) {
- throw Exception(
- "Missing configuration mapping for ${json['configuration']}");
- }
- json['configuration'] = configuration;
- json['flaky'] = false;
- json['previous_flaky'] = false;
+ await for (var results in resultsStream) {
+ for (var json in LineSplitter.split(results).map(jsonDecode)) {
+ json['build_number'] = 0;
+ json['previous_build_number'] = 0;
+ json['builder_name'] = target;
+ var configuration = configs[json['configuration']];
+ if (configuration == null) {
+ throw Exception(
+ "Missing configuration mapping for ${json['configuration']}");
+ }
+ json['configuration'] = configuration;
+ json['flaky'] = false;
+ json['previous_flaky'] = false;
- modifiedResults.writeln(jsonEncode(json));
- if (dryRun) break;
+ modifiedResults.writeln(jsonEncode(json));
+ if (dryRun) break;
+ }
}
await write(
- '$resultBase/$to/0/results.json', modifiedResults.toString(), dryRun);
- await write('$resultBase/$to/latest', '0', dryRun);
+ '$resultBase/$target/0/results.json', modifiedResults.toString(), dryRun);
+ await write('$resultBase/$target/latest', '0', dryRun);
}
Future<String> read(String url) {
diff --git a/baseline/lib/options.dart b/baseline/lib/options.dart
index ef773e1..6606483 100644
--- a/baseline/lib/options.dart
+++ b/baseline/lib/options.dart
@@ -11,6 +11,7 @@
late final List<String> builders;
late final Map<String, String> configs;
late final List<String> channels;
+ late final String target;
BaselineOptions(List<String> arguments) {
var parser = ArgParser();
@@ -23,10 +24,11 @@
abbr: 'm',
help: 'a comma separated list of configuration mappings in the form:'
'<old1>:<new1>,<old2>:<new2>');
- parser.addOption('builder-mapping',
+ parser.addMultiOption('builders',
abbr: 'b',
- help:
- 'a mapping from an old to a new builder in the form: <old>:<new>');
+ help: 'a comma separated list of builders to read result data from');
+ parser.addOption('target',
+ abbr: 't', help: 'a the name of the builder to baseline');
parser.addFlag('dry-run',
abbr: 'n',
defaultsTo: false,
@@ -35,11 +37,13 @@
parser.addFlag('help',
abbr: 'h', negatable: false, help: 'prints this message');
var parsed = parser.parse(arguments);
- if (parsed['help'] || parsed['builder-mapping'] is! String) {
+ if (parsed['help'] ||
+ parsed['builders'] is! List<String> ||
+ parsed['target'] is! String) {
print(parser.usage);
- exit(0);
+ exit(64);
}
- builders = (parsed['builder-mapping'] as String).split(':');
+ builders = (parsed['builders'] as List<String>);
configs = {
for (var v in ((parsed['config-mapping'] as Iterable<String>)
.map((c) => c.split(':'))))
@@ -47,5 +51,6 @@
};
dryRun = parsed['dry-run'];
channels = parsed['channel'];
+ target = parsed['target'];
}
}
diff --git a/baseline/pubspec.lock b/baseline/pubspec.lock
index f469ed6..fb61d12 100644
--- a/baseline/pubspec.lock
+++ b/baseline/pubspec.lock
@@ -7,14 +7,14 @@
name: _fe_analyzer_shared
url: "https://pub.dartlang.org"
source: hosted
- version: "26.0.0"
+ version: "31.0.0"
analyzer:
dependency: transitive
description:
name: analyzer
url: "https://pub.dartlang.org"
source: hosted
- version: "2.3.0"
+ version: "2.8.0"
args:
dependency: "direct main"
description:
@@ -49,7 +49,7 @@
name: cli_util
url: "https://pub.dartlang.org"
source: hosted
- version: "0.3.3"
+ version: "0.3.5"
collection:
dependency: transitive
description:
@@ -98,7 +98,7 @@
name: glob
url: "https://pub.dartlang.org"
source: hosted
- version: "2.0.1"
+ version: "2.0.2"
http_multi_server:
dependency: transitive
description:
@@ -161,7 +161,7 @@
name: mime
url: "https://pub.dartlang.org"
source: hosted
- version: "1.0.0"
+ version: "1.0.1"
node_preamble:
dependency: transitive
description:
@@ -183,15 +183,8 @@
url: "https://pub.dartlang.org"
source: hosted
version: "1.8.0"
- pedantic:
- dependency: transitive
- description:
- name: pedantic
- url: "https://pub.dartlang.org"
- source: hosted
- version: "1.11.1"
pool:
- dependency: transitive
+ dependency: "direct main"
description:
name: pool
url: "https://pub.dartlang.org"
@@ -287,21 +280,21 @@
name: test
url: "https://pub.dartlang.org"
source: hosted
- version: "1.17.12"
+ version: "1.19.5"
test_api:
dependency: transitive
description:
name: test_api
url: "https://pub.dartlang.org"
source: hosted
- version: "0.4.3"
+ version: "0.4.8"
test_core:
dependency: transitive
description:
name: test_core
url: "https://pub.dartlang.org"
source: hosted
- version: "0.4.2"
+ version: "0.4.9"
typed_data:
dependency: transitive
description:
@@ -315,14 +308,14 @@
name: vm_service
url: "https://pub.dartlang.org"
source: hosted
- version: "7.3.0"
+ version: "7.5.0"
watcher:
dependency: transitive
description:
name: watcher
url: "https://pub.dartlang.org"
source: hosted
- version: "1.0.0"
+ version: "1.0.1"
web_socket_channel:
dependency: transitive
description:
diff --git a/baseline/pubspec.yaml b/baseline/pubspec.yaml
index 269e260..231d934 100644
--- a/baseline/pubspec.yaml
+++ b/baseline/pubspec.yaml
@@ -8,6 +8,7 @@
dependencies:
args: ^2.3.0
+ pool: ^1.5.0
dev_dependencies:
io: ^1.0.3
diff --git a/baseline/test/baseline_test.dart b/baseline/test/baseline_test.dart
index f4d429c..ab9b0d4 100644
--- a/baseline/test/baseline_test.dart
+++ b/baseline/test/baseline_test.dart
@@ -2,6 +2,7 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
+import 'dart:convert';
import 'dart:io';
import 'package:baseline/baseline.dart';
@@ -9,10 +10,16 @@
import 'package:io/io.dart';
import 'package:test/test.dart';
-final String builderResults =
- File('test/data/builder/42/results.json').readAsStringSync();
-final String builderStableResults =
- File('test/data/builder-stable/12/results.json').readAsStringSync();
+final builder1Results = _readTestData('test/data/builder/42/results.json');
+final builder2Results = _readTestData('test/data/builder2/36/results.json');
+final builderStableResults =
+ _readTestData('test/data/builder-stable/12/results.json');
+final builder2StableResults =
+ _readTestData('test/data/builder2-stable/15/results.json');
+
+List<String> _readTestData(String path) {
+ return LineSplitter.split(File(path).readAsStringSync()).toList();
+}
main() {
test('run', () async {
@@ -38,7 +45,8 @@
expect(
baseline(
BaselineOptions([
- '--builder-mapping=builder:new-builder',
+ '--builders=builder1,builder2',
+ '--target=new-builder',
'--channel=main,stable',
'--config-mapping=config2:new-config2',
'--dry-run',
@@ -49,47 +57,63 @@
test('baseline dry-run', () async {
await baselineTest([
- '--builder-mapping=builder:new-builder',
+ '--builders=builder,builder2',
+ '--target=new-builder',
'--channel=main,stable',
- '--config-mapping=config1:new-config1,config2:new-config2',
+ '--config-mapping=config1:new-config1,config2:new-config2,'
+ 'config3:new-config3,config4:new-config4',
'--dry-run',
], {
- 'builder-stable/latest': '12',
+ 'builder-stable/latest': ['12'],
'builder-stable/12/results.json': builderStableResults,
- 'builder/42/results.json': builderResults,
- 'builder/latest': '42',
+ 'builder/42/results.json': builder1Results,
+ 'builder/latest': ['42'],
+ 'builder2-stable/15/results.json': builder2StableResults,
+ 'builder2-stable/latest': ['15'],
+ 'builder2/36/results.json': builder2Results,
+ 'builder2/latest': ['36'],
});
});
test('baseline', () async {
- const newBuilderStableResults = '''
-{"build_number":0,"previous_build_number":0,"builder_name":"new-builder-stable","configuration":"new-config1","test_name":"test1","result":"PASS","flaky":false,"previous_flaky":false}
-{"build_number":0,"previous_build_number":0,"builder_name":"new-builder-stable","configuration":"new-config2","test_name":"test2","result":"FAIL","flaky":false,"previous_flaky":false}
-''';
- const newBuilderResults = '''
-{"build_number":0,"previous_build_number":0,"builder_name":"new-builder","configuration":"new-config1","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}
-{"build_number":0,"previous_build_number":0,"builder_name":"new-builder","configuration":"new-config2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}
-''';
+ final newBuilderStableResults = unorderedEquals([
+ '{"build_number":0,"previous_build_number":0,"builder_name":"new-builder-stable","configuration":"new-config1","test_name":"test1","result":"PASS","flaky":false,"previous_flaky":false}',
+ '{"build_number":0,"previous_build_number":0,"builder_name":"new-builder-stable","configuration":"new-config2","test_name":"test2","result":"FAIL","flaky":false,"previous_flaky":false}',
+ '{"build_number":0,"previous_build_number":0,"builder_name":"new-builder-stable","configuration":"new-config3","test_name":"test1","result":"PASS","flaky":false,"previous_flaky":false}',
+ '{"build_number":0,"previous_build_number":0,"builder_name":"new-builder-stable","configuration":"new-config4","test_name":"test2","result":"FAIL","flaky":false,"previous_flaky":false}',
+ ]);
+ final newBuilderResults = unorderedEquals([
+ '{"build_number":0,"previous_build_number":0,"builder_name":"new-builder","configuration":"new-config1","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}',
+ '{"build_number":0,"previous_build_number":0,"builder_name":"new-builder","configuration":"new-config2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
+ '{"build_number":0,"previous_build_number":0,"builder_name":"new-builder","configuration":"new-config3","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}',
+ '{"build_number":0,"previous_build_number":0,"builder_name":"new-builder","configuration":"new-config4","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
+ ]);
await baselineTest([
- '--builder-mapping=builder:new-builder',
+ '--builders=builder,builder2',
+ '--target=new-builder',
'--channel=main,stable',
- '--config-mapping=config1:new-config1,config2:new-config2',
+ '--config-mapping=config1:new-config1,config2:new-config2,'
+ 'config3:new-config3,config4:new-config4',
], {
- 'builder-stable/latest': '12',
- 'builder-stable/12/results.json': builderStableResults,
'new-builder-stable/0/results.json': newBuilderStableResults,
- 'new-builder-stable/latest': '0',
+ 'new-builder-stable/latest': ['0'],
'new-builder/0/results.json': newBuilderResults,
- 'new-builder/latest': '0',
- 'builder/42/results.json': builderResults,
- 'builder/latest': '42',
+ 'new-builder/latest': ['0'],
+ 'builder-stable/latest': ['12'],
+ 'builder-stable/12/results.json': builderStableResults,
+ 'builder/42/results.json': builder1Results,
+ 'builder/latest': ['42'],
+ 'builder2-stable/15/results.json': builder2StableResults,
+ 'builder2-stable/latest': ['15'],
+ 'builder2/36/results.json': builder2Results,
+ 'builder2/latest': ['36'],
});
});
}
Future<void> baselineTest(
- List<String> arguments, Map<String, String> expectedFiles) async {
+ List<String> arguments, Map<String, dynamic> expectedFiles) async {
var temp = await Directory.systemTemp.createTemp();
try {
await copyPath('test/data', temp.path);
@@ -100,8 +124,8 @@
.map((e) => e.path.substring(temp.path.length + 1));
expect(files, containsAll(expectedFiles.keys));
for (var expectedFile in expectedFiles.entries) {
- var content =
- await File('${temp.path}/${expectedFile.key}').readAsString();
+ var content = LineSplitter.split(
+ await File('${temp.path}/${expectedFile.key}').readAsString());
expect(content, expectedFile.value,
reason: 'File "${expectedFile.key}" mismatch');
}
diff --git a/baseline/test/data/builder2-stable/15/results.json b/baseline/test/data/builder2-stable/15/results.json
new file mode 100644
index 0000000..ed50c1e
--- /dev/null
+++ b/baseline/test/data/builder2-stable/15/results.json
@@ -0,0 +1,2 @@
+{"build_number":15,"previous_build_number":12,"builder_name":"builder2-stable","configuration":"config3","test_name":"test1","result":"PASS","flaky":false,"previous_flaky":true}
+{"build_number":15,"previous_build_number":12,"builder_name":"builder2-stable","configuration":"config4","test_name":"test2","result":"FAIL","flaky":true,"previous_flaky":false}
diff --git a/baseline/test/data/builder2-stable/latest b/baseline/test/data/builder2-stable/latest
new file mode 100644
index 0000000..3f10ffe
--- /dev/null
+++ b/baseline/test/data/builder2-stable/latest
@@ -0,0 +1 @@
+15
\ No newline at end of file
diff --git a/baseline/test/data/builder2/36/results.json b/baseline/test/data/builder2/36/results.json
new file mode 100644
index 0000000..43b4db1
--- /dev/null
+++ b/baseline/test/data/builder2/36/results.json
@@ -0,0 +1,2 @@
+{"build_number":36,"previous_build_number":34,"builder_name":"builder2","configuration":"config3","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":true}
+{"build_number":36,"previous_build_number":34,"builder_name":"builder2","configuration":"config4","test_name":"test2","result":"PASS","flaky":true,"previous_flaky":false}
diff --git a/baseline/test/data/builder2/latest b/baseline/test/data/builder2/latest
new file mode 100644
index 0000000..dce6588
--- /dev/null
+++ b/baseline/test/data/builder2/latest
@@ -0,0 +1 @@
+36
\ No newline at end of file
diff --git a/baseline/test/options_test.dart b/baseline/test/options_test.dart
index 66d9297..906b476 100644
--- a/baseline/test/options_test.dart
+++ b/baseline/test/options_test.dart
@@ -5,7 +5,7 @@
import 'package:baseline/options.dart';
import 'package:test/test.dart';
-const _builders = '-ba:b';
+const _builders = ['-ba1,a2', '-tb'];
main() {
for (var channels in [
@@ -13,7 +13,7 @@
['dev', 'beta'],
['stable'],
]) {
- var arguments = ['-c${channels.join(',')}', _builders];
+ var arguments = ['-c${channels.join(',')}', ..._builders];
test('channels: "$arguments"', () {
var options = BaselineOptions(arguments);
expect(options.channels, channels);
@@ -21,22 +21,23 @@
}
test('builder-mapping', () {
- var options = BaselineOptions([_builders]);
- expect(options.builders, ['a', 'b']);
+ var options = BaselineOptions(_builders);
+ expect(options.builders, ['a1', 'a2']);
+ expect(options.target, 'b');
});
test('config-mapping', () {
- var options = BaselineOptions(['-mc:d,e:f', _builders]);
+ var options = BaselineOptions(['-mc:d,e:f', ..._builders]);
expect(options.configs, {'c': 'd', 'e': 'f'});
});
test('dry-run defaults to false', () {
- var options = BaselineOptions([_builders]);
+ var options = BaselineOptions(_builders);
expect(options.dryRun, false);
});
test('dry-run: true', () {
- var options = BaselineOptions(['-n', _builders]);
+ var options = BaselineOptions(['-n', ..._builders]);
expect(options.dryRun, true);
});
}
