| #!/usr/bin/env dart |
| // Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| // Update the flakiness data with a set of fresh results. |
| |
| // @dart = 2.9 |
| |
| import 'dart:convert'; |
| import 'dart:io'; |
| |
| import 'package:args/args.dart'; |
| |
| import 'package:test_runner/bot_results.dart'; |
| |
| void main(List<String> args) async { |
| final parser = ArgParser(); |
| parser.addFlag('help', help: 'Show the program usage.', negatable: false); |
| parser.addOption('input', abbr: 'i', help: 'Input flakiness file.'); |
| parser.addOption('output', abbr: 'o', help: 'Output flakiness file.'); |
| parser.addOption('build-id', help: 'Logdog ID of this buildbot run'); |
| parser.addOption('commit', help: 'Commit hash of this buildbot run'); |
| parser.addFlag('no-forgive', help: 'Don\'t remove any flaky records'); |
| |
| final options = parser.parse(args); |
| if (options['help']) { |
| print(''' |
| Usage: update_flakiness.dart [OPTION]... [RESULT-FILE]... |
| Update the flakiness data with a set of fresh results. |
| |
| The options are as follows: |
| |
| ${parser.usage}'''); |
| return; |
| } |
| final parameters = options.rest; |
| |
| // Load the existing flakiness data, if any. |
| final data = options['input'] != null |
| ? await loadResultsMap(options['input']) |
| : <String, Map<String, dynamic>>{}; |
| |
| final resultsForInactiveFlakiness = { |
| for (final flakyTest in data.keys) |
| if (data[flakyTest]['active'] == false) flakyTest: <String>{} |
| }; |
| // Incrementally update the flakiness data with each observed result. |
| for (final path in parameters) { |
| final results = await loadResults(path); |
| for (final resultObject in results) { |
| final String configuration = resultObject['configuration'] /*!*/; |
| final String name = resultObject['name'] /*!*/; |
| final String result = resultObject['result'] /*!*/; |
| final key = '$configuration:$name'; |
| resultsForInactiveFlakiness[key]?.add(result); |
| Map<String, dynamic> newMap() => {}; |
| final testData = data.putIfAbsent(key, newMap); |
| testData['configuration'] = configuration; |
| testData['name'] = name; |
| testData['expected'] = resultObject['expected']; |
| final outcomes = testData.putIfAbsent('outcomes', () => []); |
| final time = DateTime.now().toIso8601String(); |
| if (!outcomes.contains(result)) { |
| outcomes |
| ..add(result) |
| ..sort(); |
| testData['last_new_result_seen'] = time; |
| } |
| if (testData['current'] == result) { |
| testData['current_counter']++; |
| } else { |
| testData['current'] = result; |
| testData['current_counter'] = 1; |
| } |
| final occurrences = testData.putIfAbsent('occurrences', newMap); |
| occurrences.putIfAbsent(result, () => 0); |
| occurrences[result]++; |
| final firstSeen = testData.putIfAbsent('first_seen', newMap); |
| firstSeen.putIfAbsent(result, () => time); |
| final lastSeen = testData.putIfAbsent('last_seen', newMap); |
| lastSeen[result] = time; |
| final matches = testData.putIfAbsent('matches', newMap); |
| matches[result] = resultObject['matches']; |
| |
| if (options['build-id'] != null) { |
| final buildIds = testData.putIfAbsent('build_ids', newMap); |
| buildIds[result] = options['build-id']; |
| } |
| if (options['commit'] != null) { |
| final commits = testData.putIfAbsent('commits', newMap); |
| commits[result] = options['commit']; |
| } |
| } |
| } |
| |
| // Write out the new flakiness data. |
| final sink = |
| options['output'] != null ? File(options['output']).openWrite() : stdout; |
| final keys = data.keys.toList()..sort(); |
| for (final key in keys) { |
| final testData = data[key]; |
| if (testData['outcomes'].length < 2) continue; |
| // Reactivate inactive flaky results that are flaky again. |
| if (testData['active'] == false) { |
| if (resultsForInactiveFlakiness[key].length > 1) { |
| testData['active'] = true; |
| testData['reactivation_count'] = |
| (testData['reactivation_count'] ?? 0) + 1; |
| } |
| } else if (!options['no-forgive'] && testData['current_counter'] >= 100) { |
| // Forgive tests that have been stable for 100 builds. |
| testData['active'] = false; |
| } else { |
| testData['active'] = true; |
| } |
| |
| sink.writeln(jsonEncode(testData)); |
| } |
| } |