[baseline] Support baselining release builders from main
Example (baseline vm-linux-(dev|beta|stable) with vm-linux results):
bin/baseline.dart -n -tvm-linux -cdev,beta,stable
Bug: b/201272359
Change-Id: Ifd581d8ab1e86a625070d2c41be1ba141aa6d12f
Reviewed-on: https://dart-review.googlesource.com/c/dart_ci/+/279690
Reviewed-by: William Hesse <whesse@google.com>
Commit-Queue: Alexander Thomas <athom@google.com>
diff --git a/baseline/bin/baseline.dart b/baseline/bin/baseline.dart
index bc549fc..607689f 100755
--- a/baseline/bin/baseline.dart
+++ b/baseline/bin/baseline.dart
@@ -7,5 +7,5 @@
import 'package:baseline/options.dart';
void main(List<String> arguments) async {
- await baseline(BaselineOptions(arguments));
+ await baseline(BaselineOptions.parse(arguments));
}
diff --git a/baseline/lib/baseline.dart b/baseline/lib/baseline.dart
index b68bfd7..fd8653b 100644
--- a/baseline/lib/baseline.dart
+++ b/baseline/lib/baseline.dart
@@ -17,6 +17,8 @@
await Future.wait([
for (final channel in options.channels)
if (channel == 'main')
+ // baseline a new builder on main
+ // builder,builder2 -> new-builder
baselineBuilder(
options.builders,
channel,
@@ -24,9 +26,23 @@
options.target,
options.configs,
options.dryRun,
- options.ignoreUnmapped,
+ options.mapping,
+ resultBase)
+ else if (options.builders.contains(options.target))
+ // baseline a builder on a channel with main builder data
+ // builder,builder2 -> builder-dev
+ baselineBuilder(
+ options.builders,
+ channel,
+ options.suites,
+ '${options.target}-$channel',
+ options.configs,
+ options.dryRun,
+ options.mapping,
resultBase)
else
+ // baseline a builder on a channel with channel builder data
+ // builder-dev,builder2-dev -> new-builder-dev
baselineBuilder(
options.builders.map((b) => '$b-$channel').toList(),
channel,
@@ -34,7 +50,7 @@
'${options.target}-$channel',
options.configs,
options.dryRun,
- options.ignoreUnmapped,
+ options.mapping,
resultBase)
]);
}
@@ -46,7 +62,7 @@
String target,
Map<String, String> configs,
bool dryRun,
- bool ignoreUnmapped,
+ ConfigurationMapping mapping,
String resultBase) async {
var resultsStream = Pool(4).forEach(builders, (builder) async {
var latest = await read('$resultBase/builders/$builder/latest');
@@ -55,14 +71,12 @@
var modifiedResults = StringBuffer();
var modifiedResultsPerConfig = <String, StringBuffer>{};
await for (var results in resultsStream) {
- for (var json in LineSplitter.split(results).map(jsonDecode)) {
- var configuration = configs[json['configuration']];
+ for (var json in LineSplitter.split(results)
+ .map(jsonDecode)
+ .cast<Map<String, dynamic>>()) {
+ var configuration = mapping(json['configuration'], configs);
if (configuration == null) {
- if (ignoreUnmapped) {
- continue;
- }
- throw Exception(
- "Missing configuration mapping for ${json['configuration']}");
+ continue;
}
if (suites.isNotEmpty && !suites.contains(json['suite'])) continue;
json['configuration'] = configuration;
diff --git a/baseline/lib/options.dart b/baseline/lib/options.dart
index df8ed0f..d4aea52 100644
--- a/baseline/lib/options.dart
+++ b/baseline/lib/options.dart
@@ -7,15 +7,18 @@
import 'package:args/args.dart';
class BaselineOptions {
- late final bool ignoreUnmapped;
- late final bool dryRun;
- late final List<String> builders;
- late final Map<String, String> configs;
- late final List<String> channels;
- late final Set<String> suites;
- late final String target;
+ final List<String> builders;
+ final Map<String, String> configs;
+ final bool dryRun;
+ final List<String> channels;
+ final ConfigurationMapping mapping;
+ final Set<String> suites;
+ final String target;
- BaselineOptions(List<String> arguments) {
+ BaselineOptions(this.builders, this.configs, this.dryRun, this.channels,
+ this.mapping, this.suites, this.target);
+
+ factory BaselineOptions.parse(List<String> arguments) {
var parser = ArgParser();
parser.addMultiOption('channel',
abbr: 'c',
@@ -24,6 +27,7 @@
help: 'a comma separated list of channels');
parser.addMultiOption('config-mapping',
abbr: 'm',
+ defaultsTo: ['*'],
help: 'a comma separated list of configuration mappings in the form:'
'<old1>:<new1>,<old2>:<new2>');
parser.addMultiOption('builders',
@@ -43,7 +47,6 @@
defaultsTo: false,
help: 'ignore tests in unmapped configurations',
negatable: false);
-
parser.addFlag('help',
abbr: 'h', negatable: false, help: 'prints this message');
var parsed = parser.parse(arguments);
@@ -53,16 +56,49 @@
print(parser.usage);
exit(64);
}
- builders = (parsed['builders'] as List<String>);
- configs = {
- for (var v in ((parsed['config-mapping'] as Iterable<String>)
- .map((c) => c.split(':'))))
- v[0]: v[1]
- };
- ignoreUnmapped = parsed['ignore-unmapped'];
- dryRun = parsed['dry-run'];
- channels = parsed['channel'];
- suites = Set.unmodifiable(parsed['suites']);
- target = parsed['target'];
+ var builders = (parsed['builders'] as List<String>);
+ final target = parsed['target'];
+ if (builders.isEmpty) {
+ builders = [target];
+ }
+ var mapping = parsed['ignore-unmapped']
+ ? ConfigurationMapping.relaxed
+ : ConfigurationMapping.strict;
+ var configs = const <String, String>{};
+ final configMapping = parsed['config-mapping'] as List<String>;
+ if (configMapping.length == 1 && configMapping.first == '*') {
+ mapping = ConfigurationMapping.none;
+ } else {
+ configs = {
+ for (var v in configMapping.map((c) => c.split(':'))) v[0]: v[1]
+ };
+ }
+ final dryRun = parsed['dry-run'];
+ final channels = parsed['channel'];
+ final suites = Set.unmodifiable(parsed['suites'] as List<String>);
+
+ return BaselineOptions(
+ builders, configs, dryRun, channels, mapping, suites, target);
}
}
+
+String? _strict(String configuration, Map<String, String> configs) =>
+ configs[configuration] ??
+ (throw Exception("Missing configuration mapping for $configuration"));
+String? _relaxed(String configuration, Map<String, String> configs) =>
+ configs[configuration];
+String? _none(String configuration, Map<String, String> configs) =>
+ configuration;
+
+enum ConfigurationMapping {
+ none(_none),
+ strict(_strict),
+ relaxed(_relaxed);
+
+ const ConfigurationMapping(this.mapping);
+ final String? Function(String configuration, Map<String, String> configs)
+ mapping;
+
+ String? call(String configuration, Map<String, String> configs) =>
+ mapping(configuration, configs);
+}
diff --git a/baseline/test/baseline_test.dart b/baseline/test/baseline_test.dart
index 0694b5c..aca7363 100644
--- a/baseline/test/baseline_test.dart
+++ b/baseline/test/baseline_test.dart
@@ -56,7 +56,7 @@
test('baseline missing config mapping throws', () {
expect(
baseline(
- BaselineOptions([
+ BaselineOptions.parse([
'--builders=builder,builder2',
'--target=new-builder',
'--channel=main,stable',
@@ -103,6 +103,50 @@
});
});
+ test('baseline default config mapping', () async {
+ final newBuilderDevResults = [
+ '{"build_number":"0","previous_build_number":"0","builder_name":"builder-dev","configuration":"config1","suite":"suite1","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}',
+ '{"build_number":"0","previous_build_number":"0","builder_name":"builder-dev","configuration":"config2","suite":"suite2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
+ '{"build_number":"0","previous_build_number":"0","builder_name":"builder-dev","configuration":"config3","suite":"suite1","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}',
+ '{"build_number":"0","previous_build_number":"0","builder_name":"builder-dev","configuration":"config4","suite":"suite2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
+ ];
+ final newBuilderStableResults = [
+ '{"build_number":"0","previous_build_number":"0","builder_name":"builder-stable","configuration":"config1","suite":"suite1","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}',
+ '{"build_number":"0","previous_build_number":"0","builder_name":"builder-stable","configuration":"config2","suite":"suite2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
+ '{"build_number":"0","previous_build_number":"0","builder_name":"builder-stable","configuration":"config3","suite":"suite1","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}',
+ '{"build_number":"0","previous_build_number":"0","builder_name":"builder-stable","configuration":"config4","suite":"suite2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
+ ];
+ await baselineTest([
+ '--builders=builder,builder2',
+ '--target=builder',
+ '--channel=dev,stable',
+ ], {
+ ...testData,
+ 'builders/builder-stable/0/results.json':
+ unorderedEquals(newBuilderStableResults),
+ 'builders/builder-stable/latest': ['0'],
+ 'configuration/stable/config1/0/results.json': [
+ newBuilderStableResults[0]
+ ],
+ 'configuration/stable/config2/0/results.json': [
+ newBuilderStableResults[1]
+ ],
+ 'configuration/stable/config3/0/results.json': [
+ newBuilderStableResults[2]
+ ],
+ 'configuration/stable/config4/0/results.json': [
+ newBuilderStableResults[3]
+ ],
+ 'builders/builder-dev/0/results.json':
+ unorderedEquals(newBuilderDevResults),
+ 'builders/builder-dev/latest': ['0'],
+ 'configuration/dev/config1/0/results.json': [newBuilderDevResults[0]],
+ 'configuration/dev/config2/0/results.json': [newBuilderDevResults[1]],
+ 'configuration/dev/config3/0/results.json': [newBuilderDevResults[2]],
+ 'configuration/dev/config4/0/results.json': [newBuilderDevResults[3]],
+ });
+ });
+
test('baseline dry-run', () async {
await baselineTest([
'--builders=builder,builder2',
@@ -200,7 +244,7 @@
var temp = await Directory.systemTemp.createTemp();
try {
await copyPath('test/data', temp.path);
- await baseline(BaselineOptions(arguments), temp.path);
+ await baseline(BaselineOptions.parse(arguments), temp.path);
var files = temp
.listSync(recursive: true)
.whereType<File>()
diff --git a/baseline/test/options_test.dart b/baseline/test/options_test.dart
index 856758e..ed9d9c1 100644
--- a/baseline/test/options_test.dart
+++ b/baseline/test/options_test.dart
@@ -15,39 +15,67 @@
]) {
var arguments = ['-c${channels.join(',')}', ..._builders];
test('channels: "$arguments"', () {
- var options = BaselineOptions(arguments);
+ var options = BaselineOptions.parse(arguments);
expect(options.channels, channels);
});
}
test('builder-mapping', () {
- var options = BaselineOptions(_builders);
+ var options = BaselineOptions.parse(_builders);
expect(options.builders, ['a1', 'a2']);
expect(options.target, 'b');
});
+ test('builder-mapping: default to target', () {
+ var options = BaselineOptions.parse(['-tb']);
+ expect(options.builders, ['b']);
+ });
+
test('config-mapping', () {
- var options = BaselineOptions(['-mc:d,e:f', ..._builders]);
+ var options = BaselineOptions.parse(['-mc:d,e:f', ..._builders]);
expect(options.configs, {'c': 'd', 'e': 'f'});
+ expect(options.mapping, ConfigurationMapping.strict);
+ });
+
+ test('config-mapping: star', () {
+ var options = BaselineOptions.parse(['-m*', ..._builders]);
+ expect(options.configs, const {});
+ expect(options.mapping, ConfigurationMapping.none);
+ });
+
+ test('config-mapping: ignore-unmapped', () {
+ var options = BaselineOptions.parse(['-u', '-mc:d,e:f', ..._builders]);
+ expect(options.mapping, ConfigurationMapping.relaxed);
});
test('suites', () {
- var options = BaselineOptions(['-ss1,s2', ..._builders]);
+ var options = BaselineOptions.parse(['-ss1,s2', ..._builders]);
expect(options.suites, {'s1', 's2'});
});
- test('ignore-unmapped', () {
- var options = BaselineOptions(['-u', '-mc:d,e:f', ..._builders]);
- expect(options.ignoreUnmapped, true);
- });
-
test('dry-run defaults to false', () {
- var options = BaselineOptions(_builders);
+ var options = BaselineOptions.parse(_builders);
expect(options.dryRun, false);
});
test('dry-run: true', () {
- var options = BaselineOptions(['-n', ..._builders]);
+ var options = BaselineOptions.parse(['-n', ..._builders]);
expect(options.dryRun, true);
});
+
+ test('mapping: strict', () {
+ expect(ConfigurationMapping.strict('foo', {'foo': 'bar'}), 'bar');
+ expect(() => ConfigurationMapping.strict('oof', {'foo': 'bar'}),
+ throwsException);
+ });
+
+ test('mapping: relaxed', () {
+ expect(ConfigurationMapping.relaxed('foo', {'foo': 'bar'}), 'bar');
+ expect(ConfigurationMapping.relaxed('oof', {'foo': 'bar'}), null);
+ });
+
+ test('mapping: none', () {
+ expect(ConfigurationMapping.none('foo', {'foo': 'bar'}), 'foo');
+ expect(ConfigurationMapping.none('oof', {'foo': 'bar'}), 'oof');
+ });
}
