[baseline] Upload per-configuration results
Change-Id: Ic018ac2a2ba2e2ff69db5ca406edb3c24393890b
Reviewed-on: https://dart-review.googlesource.com/c/dart_ci/+/251220
Commit-Queue: Alexander Thomas <athom@google.com>
Reviewed-by: William Hesse <whesse@google.com>
diff --git a/baseline/lib/baseline.dart b/baseline/lib/baseline.dart
index a8cd9e7..21278f5 100644
--- a/baseline/lib/baseline.dart
+++ b/baseline/lib/baseline.dart
@@ -8,7 +8,7 @@
import 'package:pool/pool.dart';
-const _resultBase = 'gs://dart-test-results/builders';
+const _resultBase = 'gs://dart-test-results';
/// Baselines a builder with the [options] and copies the results to the
/// [resultBase]. [resultBase] can be a URL or a path supported by `gsutil cp`.
@@ -17,11 +17,12 @@
await Future.wait([
for (final channel in options.channels)
if (channel == 'main')
- baselineBuilder(options.builders, options.target, options.configs,
- options.dryRun, options.ignoreUnmapped, resultBase)
+ baselineBuilder(options.builders, channel, options.target,
+ options.configs, options.dryRun, options.ignoreUnmapped, resultBase)
else
baselineBuilder(
options.builders.map((b) => '$b-$channel').toList(),
+ channel,
'${options.target}-$channel',
options.configs,
options.dryRun,
@@ -32,16 +33,18 @@
Future<void> baselineBuilder(
List<String> builders,
+ String channel,
String target,
Map<String, String> configs,
bool dryRun,
bool ignoreUnmapped,
String resultBase) async {
var resultsStream = Pool(4).forEach(builders, (builder) async {
- var latest = await read('$resultBase/$builder/latest');
- return await read('$resultBase/$builder/$latest/results.json');
+ var latest = await read('$resultBase/builders/$builder/latest');
+ return await read('$resultBase/builders/$builder/$latest/results.json');
});
var modifiedResults = StringBuffer();
+ var modifiedResultsPerConfig = <String, StringBuffer>{};
await for (var results in resultsStream) {
for (var json in LineSplitter.split(results).map(jsonDecode)) {
var configuration = configs[json['configuration']];
@@ -59,13 +62,23 @@
json['flaky'] = false;
json['previous_flaky'] = false;
- modifiedResults.writeln(jsonEncode(json));
+ var encoded = jsonEncode(json);
+ modifiedResults.writeln(encoded);
+ modifiedResultsPerConfig
+ .putIfAbsent(configuration, () => StringBuffer())
+ .writeln(encoded);
if (dryRun) break;
}
}
- await write(
- '$resultBase/$target/0/results.json', modifiedResults.toString(), dryRun);
- await write('$resultBase/$target/latest', '0', dryRun);
+ await write('$resultBase/builders/$target/0/results.json',
+ modifiedResults.toString(), dryRun);
+ for (var entry in modifiedResultsPerConfig.entries) {
+ await write(
+ '$resultBase/configuration/$channel/${entry.key}/0/results.json',
+ entry.value.toString(),
+ dryRun);
+ }
+ await write('$resultBase/builders/$target/latest', '0', dryRun);
}
Future<String> read(String url) {
diff --git a/baseline/test/baseline_test.dart b/baseline/test/baseline_test.dart
index 67b54e4..8359d57 100644
--- a/baseline/test/baseline_test.dart
+++ b/baseline/test/baseline_test.dart
@@ -10,21 +10,23 @@
import 'package:io/io.dart';
import 'package:test/test.dart';
-final builder1Results = _readTestData('test/data/builder/42/results.json');
-final builder2Results = _readTestData('test/data/builder2/36/results.json');
+final builder1Results =
+ _readTestData('test/data/builders/builder/42/results.json');
+final builder2Results =
+ _readTestData('test/data/builders/builder2/36/results.json');
final builderStableResults =
- _readTestData('test/data/builder-stable/12/results.json');
+ _readTestData('test/data/builders/builder-stable/12/results.json');
final builder2StableResults =
- _readTestData('test/data/builder2-stable/15/results.json');
+ _readTestData('test/data/builders/builder2-stable/15/results.json');
final testData = {
- 'builder-stable/latest': ['12'],
- 'builder-stable/12/results.json': builderStableResults,
- 'builder/42/results.json': builder1Results,
- 'builder/latest': ['42'],
- 'builder2-stable/15/results.json': builder2StableResults,
- 'builder2-stable/latest': ['15'],
- 'builder2/36/results.json': builder2Results,
- 'builder2/latest': ['36'],
+ 'builders/builder-stable/latest': ['12'],
+ 'builders/builder-stable/12/results.json': builderStableResults,
+ 'builders/builder/42/results.json': builder1Results,
+ 'builders/builder/latest': ['42'],
+ 'builders/builder2-stable/15/results.json': builder2StableResults,
+ 'builders/builder2-stable/latest': ['15'],
+ 'builders/builder2/36/results.json': builder2Results,
+ 'builders/builder2/latest': ['36'],
};
List<String> _readTestData(String path) {
@@ -77,12 +79,12 @@
});
test('baseline ignored config mapping', () async {
- final newBuilderStableResults = unorderedEquals([
- '{"build_number":"0","previous_build_number":"0","builder_name":"new-builder-stable","configuration":"new-config2","test_name":"test2","result":"FAIL","flaky":false,"previous_flaky":false}',
- ]);
- final newBuilderResults = unorderedEquals([
- '{"build_number":"0","previous_build_number":"0","builder_name":"new-builder","configuration":"new-config2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
- ]);
+ final newBuilderStableResults = [
+ '{"build_number":"0","previous_build_number":"0","builder_name":"new-builder-stable","configuration":"new-config2","test_name":"test2","result":"FAIL","flaky":false,"previous_flaky":false}'
+ ];
+ final newBuilderResults = [
+ '{"build_number":"0","previous_build_number":"0","builder_name":"new-builder","configuration":"new-config2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}'
+ ];
await baselineTest([
'--builders=builder,builder2',
'--target=new-builder',
@@ -90,10 +92,13 @@
'--config-mapping=config2:new-config2',
'--ignore-unmapped',
], {
- 'new-builder-stable/0/results.json': newBuilderStableResults,
- 'new-builder-stable/latest': ['0'],
- 'new-builder/0/results.json': newBuilderResults,
- 'new-builder/latest': ['0'],
+ 'builders/new-builder-stable/0/results.json': newBuilderStableResults,
+ 'builders/new-builder-stable/latest': ['0'],
+ 'builders/new-builder/0/results.json': newBuilderResults,
+ 'builders/new-builder/latest': ['0'],
+ 'configuration/main/new-config2/0/results.json': newBuilderResults,
+ 'configuration/stable/new-config2/0/results.json':
+ newBuilderStableResults,
...testData,
});
});
@@ -110,18 +115,18 @@
});
test('baseline', () async {
- final newBuilderStableResults = unorderedEquals([
+ final newBuilderStableResults = [
'{"build_number":"0","previous_build_number":"0","builder_name":"new-builder-stable","configuration":"new-config1","test_name":"test1","result":"PASS","flaky":false,"previous_flaky":false}',
'{"build_number":"0","previous_build_number":"0","builder_name":"new-builder-stable","configuration":"new-config2","test_name":"test2","result":"FAIL","flaky":false,"previous_flaky":false}',
'{"build_number":"0","previous_build_number":"0","builder_name":"new-builder-stable","configuration":"new-config3","test_name":"test1","result":"PASS","flaky":false,"previous_flaky":false}',
'{"build_number":"0","previous_build_number":"0","builder_name":"new-builder-stable","configuration":"new-config4","test_name":"test2","result":"FAIL","flaky":false,"previous_flaky":false}',
- ]);
- final newBuilderResults = unorderedEquals([
+ ];
+ final newBuilderResults = [
'{"build_number":"0","previous_build_number":"0","builder_name":"new-builder","configuration":"new-config1","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}',
'{"build_number":"0","previous_build_number":"0","builder_name":"new-builder","configuration":"new-config2","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
'{"build_number":"0","previous_build_number":"0","builder_name":"new-builder","configuration":"new-config3","test_name":"test1","result":"FAIL","flaky":false,"previous_flaky":false}',
'{"build_number":"0","previous_build_number":"0","builder_name":"new-builder","configuration":"new-config4","test_name":"test2","result":"PASS","flaky":false,"previous_flaky":false}',
- ]);
+ ];
await baselineTest([
'--builders=builder,builder2',
@@ -130,10 +135,27 @@
'--config-mapping=config1:new-config1,config2:new-config2,'
'config3:new-config3,config4:new-config4',
], {
- 'new-builder-stable/0/results.json': newBuilderStableResults,
- 'new-builder-stable/latest': ['0'],
- 'new-builder/0/results.json': newBuilderResults,
- 'new-builder/latest': ['0'],
+ 'builders/new-builder-stable/0/results.json':
+ unorderedEquals(newBuilderStableResults),
+ 'builders/new-builder-stable/latest': ['0'],
+ 'builders/new-builder/0/results.json': unorderedEquals(newBuilderResults),
+ 'builders/new-builder/latest': ['0'],
+ 'configuration/main/new-config1/0/results.json': [newBuilderResults[0]],
+ 'configuration/main/new-config2/0/results.json': [newBuilderResults[1]],
+ 'configuration/main/new-config3/0/results.json': [newBuilderResults[2]],
+ 'configuration/main/new-config4/0/results.json': [newBuilderResults[3]],
+ 'configuration/stable/new-config1/0/results.json': [
+ newBuilderStableResults[0]
+ ],
+ 'configuration/stable/new-config2/0/results.json': [
+ newBuilderStableResults[1]
+ ],
+ 'configuration/stable/new-config3/0/results.json': [
+ newBuilderStableResults[2]
+ ],
+ 'configuration/stable/new-config4/0/results.json': [
+ newBuilderStableResults[3]
+ ],
...testData,
});
});
diff --git a/baseline/test/data/builder-stable/12/results.json b/baseline/test/data/builders/builder-stable/12/results.json
similarity index 100%
rename from baseline/test/data/builder-stable/12/results.json
rename to baseline/test/data/builders/builder-stable/12/results.json
diff --git a/baseline/test/data/builder-stable/latest b/baseline/test/data/builders/builder-stable/latest
similarity index 100%
rename from baseline/test/data/builder-stable/latest
rename to baseline/test/data/builders/builder-stable/latest
diff --git a/baseline/test/data/builder/42/results.json b/baseline/test/data/builders/builder/42/results.json
similarity index 100%
rename from baseline/test/data/builder/42/results.json
rename to baseline/test/data/builders/builder/42/results.json
diff --git a/baseline/test/data/builder/latest b/baseline/test/data/builders/builder/latest
similarity index 100%
rename from baseline/test/data/builder/latest
rename to baseline/test/data/builders/builder/latest
diff --git a/baseline/test/data/builder2-stable/15/results.json b/baseline/test/data/builders/builder2-stable/15/results.json
similarity index 100%
rename from baseline/test/data/builder2-stable/15/results.json
rename to baseline/test/data/builders/builder2-stable/15/results.json
diff --git a/baseline/test/data/builder2-stable/latest b/baseline/test/data/builders/builder2-stable/latest
similarity index 100%
rename from baseline/test/data/builder2-stable/latest
rename to baseline/test/data/builders/builder2-stable/latest
diff --git a/baseline/test/data/builder2/36/results.json b/baseline/test/data/builders/builder2/36/results.json
similarity index 100%
rename from baseline/test/data/builder2/36/results.json
rename to baseline/test/data/builders/builder2/36/results.json
diff --git a/baseline/test/data/builder2/latest b/baseline/test/data/builders/builder2/latest
similarity index 100%
rename from baseline/test/data/builder2/latest
rename to baseline/test/data/builders/builder2/latest
