blob: 278f7164d6795e6d9557acb33acdc87a65ac782d [file] [log] [blame] [edit]
#!/usr/bin/env dart
// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// Update the flakiness data with a set of fresh results.
import 'dart:convert';
import 'dart:io';
import 'package:args/args.dart';
import 'package:test_runner/bot_results.dart';
void main(List<String> args) async {
final parser = ArgParser();
parser.addFlag('help', help: 'Show the program usage.', negatable: false);
parser.addOption('input', abbr: 'i', help: 'Input flakiness file.');
parser.addOption('output', abbr: 'o', help: 'Output flakiness file.');
parser.addOption('build-id', help: 'Logdog ID of this buildbot run');
parser.addOption('commit', help: 'Commit hash of this buildbot run');
parser.addFlag('no-forgive', help: 'Don\'t remove any flaky records');
final options = parser.parse(args);
if (options.flag('help')) {
print('''
Usage: update_flakiness.dart [OPTION]... [RESULT-FILE]...
Update the flakiness data with a set of fresh results.
The options are as follows:
${parser.usage}''');
return;
}
final parameters = options.rest;
// Load the existing flakiness data, if any.
final data = options.option('input') != null
? await loadResultsMap(options.option('input')!)
: <String, Map<String, dynamic>>{};
final resultsForInactiveFlakiness = {
for (final flakyTest in data.keys)
if (data[flakyTest]!['active'] == false) flakyTest: <String>{}
};
final now = DateTime.now();
final nowString = now.toIso8601String();
// Incrementally update the flakiness data with each observed result.
for (final path in parameters) {
final results = await loadResults(path);
for (final resultObject in results) {
final String configuration = resultObject['configuration'];
final String name = resultObject['name'];
final String result = resultObject['result'];
final key = '$configuration:$name';
resultsForInactiveFlakiness[key]?.add(result);
final testData = data[key] ??= {};
testData['configuration'] = configuration;
testData['name'] = name;
testData['expected'] = resultObject['expected'];
final List<dynamic> outcomes = testData['outcomes'] ??= <dynamic>[];
if (!outcomes.contains(result)) {
outcomes
..add(result)
..sort();
testData['last_new_result_seen'] = nowString;
}
if (testData['current'] == result) {
var currentCounter = testData['current_counter'] as int;
testData['current_counter'] = currentCounter + 1;
} else {
testData['current'] = result;
testData['current_counter'] = 1;
}
Map<String, dynamic> mapField(String key) =>
testData[key] ??= <String, dynamic>{};
mapField('occurrences')[result] =
(mapField('occurrences')[result] as int? ?? 0) + 1;
mapField('first_seen')[result] ??= nowString;
mapField('last_seen')[result] = nowString;
mapField('matches')[result] = resultObject['matches'];
if (options.option('build-id') != null) {
mapField('build_ids')[result] = options.option('build-id')!;
}
if (options.option('commit') != null) {
mapField('commits')[result] = options.option('commit')!;
}
}
}
// Write out the new flakiness data.
final flakinessHorizon = now.subtract(Duration(days: 7));
final sink = options.option('output') != null
? File(options.option('output')!).openWrite()
: stdout;
final keys = data.keys.toList()..sort();
for (final key in keys) {
final testData = data[key]!;
if ((testData['outcomes'] as List).length < 2) continue;
// Reactivate inactive flaky results that are flaky again.
if (testData['active'] == false) {
if (resultsForInactiveFlakiness[key]!.length > 1) {
testData['active'] = true;
testData['reactivation_count'] =
(testData['reactivation_count'] as int? ?? 0) + 1;
}
} else if (options.flag('no-forgive')) {
testData['active'] = true;
} else if (testData['current_counter'] as int >= 100) {
// Forgive tests that have been stable for 100 builds.
testData['active'] = false;
} else {
// Forgive tests that have been stable since flakiness horizon (one week).
final resultTimes = [
for (final timeString in (testData['last_seen'] as Map).values)
DateTime.parse(timeString)
]..sort();
// The latest timestamp is the current result. The one before that is the
// timestamp of the latest flake. Timestamps are sorted from earliest to latest.
if (resultTimes[resultTimes.length - 2].isBefore(flakinessHorizon)) {
testData['active'] = false;
} else {
testData['active'] = true;
}
}
sink.writeln(jsonEncode(testData));
}
}