blob: a9fa2d49aee280dd5cc2bbfa8dc1ee2f62415337 [file] [log] [blame]
// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// This script is used by the bisection mechanism to update the blamelists
// of active, non-approved failures which include the commit of the current
// bisection build.
import 'dart:io';
import 'package:args/args.dart';
import 'package:test_runner/bot_results.dart';
import 'lib/src/firestore.dart';
const newTest = 'new test';
const skippedTest = 'skipped';
const maxAttempts = 20;
late FirestoreDatabase database;
class ResultRecord {
final Map<String, dynamic> data;
ResultRecord(this.data);
Map field(String name) => (data['fields'] as Map)[name];
int get blamelistStartIndex {
return int.parse(field('blamelist_start_index')['integerValue']);
}
set blamelistStartIndex(int index) {
field('blamelist_start_index')['integerValue'] = '$index';
}
int get blamelistEndIndex {
return int.parse(field('blamelist_end_index')['integerValue']);
}
String get result => field('result')['stringValue'] /*!*/;
String get previousResult => field('previous_result')['stringValue'] /*!*/;
String get name => field('name')['stringValue'] /*!*/;
String get updateTime => data['updateTime'] /*!*/;
}
Query unapprovedActiveFailuresQuery(String configuration) {
return Query(
'results',
CompositeFilter('AND', [
Field('approved').equals(Value.boolean(false)),
// TODO(karlklose): also search for inactive failures?
Field('active_configurations').contains(Value.string(configuration)),
// TODO(karlklose): add index to check for blamelist_start_index < ?
]));
}
Future<int> getCommitIndex(String commit) async {
try {
var document =
(await database.getDocument('commits', commit)).cast<String, dynamic>();
var index = (document['fields'] as Map)['index'] as Map;
if (index['integerValue'] == null) {
throw Exception('Expected an integer, but got "$index"');
}
return int.parse(index['integerValue']);
} catch (exception) {
print('Could not retrieve index for commit "$commit".\n');
rethrow;
}
}
/// Compute if the record should be updated based on the outcomes in the
/// result record and the new test result.
bool shouldUpdateRecord(ResultRecord resultRecord, Result? testResult) {
if (testResult == null || !testResult.matches) {
return false;
}
var baseline = testResult.expectation.toLowerCase();
if (resultRecord.previousResult.toLowerCase() != baseline) {
// Currently we only support the case where a bisection run improves the
// accuracy of a "Green" -> "Red" result record.
return false;
}
if (resultRecord.result.toLowerCase() == newTest ||
resultRecord.result.toLowerCase() == skippedTest) {
// Skipped tests are often configuration dependent, so it could be wrong
// to generalize their effect for the result record to different
// configurations.
return false;
}
return true;
}
Future<void> updateBlameLists(String configuration, String commit,
Map<String, Map<String, dynamic>> testResults) async {
int commitIndex = await getCommitIndex(commit);
var query = unapprovedActiveFailuresQuery(configuration);
bool needsRetry;
int attempts = 0;
do {
needsRetry = false;
var documentPaths = (await database.runQuery(query))
.cast<Map>()
.where((result) => result['document'] != null)
.map((result) => (result['document'] as Map)['name'] as String);
for (var documentPath in documentPaths) {
database.beginTransaction();
var documentName = documentPath.split('/').last;
final docMap = await database.getDocument('results', documentName);
var result = ResultRecord(docMap.cast<String, dynamic>());
if (commitIndex < result.blamelistStartIndex ||
commitIndex >= result.blamelistEndIndex) {
continue;
}
String name = result.name;
var testResultData = testResults['$configuration:$name'];
var testResult =
testResultData != null ? Result.fromMap(testResultData) : null;
if (!shouldUpdateRecord(result, testResult)) {
continue;
}
print('Found result record: $configuration:${result.name}: '
'${result.previousResult} -> ${result.result} '
'in ${result.blamelistStartIndex}..${result.blamelistEndIndex} '
'to update with ${testResult?.outcome} at $commitIndex.');
// We found a result representation for this test and configuration whose
// blamelist includes this results' commit but whose outcome is different
// then the outcome in the provided test results.
// This means that this commit should not be part of the result
// representation and we can update the lower bound of the commit range
// and the previous result.
var newStartIndex = commitIndex + 1;
if (newStartIndex > result.blamelistEndIndex) {
print('internal error: inconsistent results; skipping results entry');
continue;
}
result.blamelistStartIndex = newStartIndex;
var updateIndex = Update(['blamelist_start_index'], result.data);
if (!await database.commit(writes: [updateIndex])) {
// Committing the change to the database had a conflict, retry.
needsRetry = true;
if (++attempts == maxAttempts) {
throw Exception('Exceeded maximum retry attempts ($maxAttempts).');
}
print('Transaction failed, trying again!');
}
}
} while (needsRetry);
}
void main(List<String> arguments) async {
var parser = ArgParser()
..addOption('auth-token',
abbr: 'a',
help: 'path to a file containing the gcloud auth token (required)')
..addOption('results',
abbr: 'r',
help: 'path to a file containing the test results (required)')
..addFlag('staging', abbr: 's', help: 'use staging database');
var options = parser.parse(arguments);
if (options.rest.isNotEmpty ||
options.option('results') == null ||
options.option('auth-token') == null) {
print(parser.usage);
exit(1);
}
var results = await loadResultsMap(options.option('results')!);
if (results.isEmpty) {
print("No test results provided, nothing to update.");
return;
}
// Pick an arbitrary result entry to find configuration and commit hash.
var firstResult = Result.fromMap(results.values.first);
var commit = firstResult.commitHash!;
var configuration = firstResult.configuration;
var project = options.flag('staging') ? 'dart-ci-staging' : 'dart-ci';
database = FirestoreDatabase(
project, await readGcloudAuthToken(options.option('auth-token')!));
await updateBlameLists(configuration, commit, results);
database.closeClient();
}