blob: c02d3e79dcaae127f5e903d7f783e57fd0e5f6ee [file] [log] [blame]
// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// Run tests like on the given builder and/or named configuration.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:args/args.dart';
import 'package:smith/smith.dart';
import 'bot_results.dart';
import 'src/options.dart';
const int deflakingCount = 5;
/// Quotes a string in shell single quote mode. This function produces a single
/// shell argument that evaluates to the exact string provided, handling any
/// special characters in the input string. Shell single quote mode works uses
/// the single quote character as the delimiter and uses the characters
/// in-between verbatim without any special processing. To insert the single
/// quote character itself, escape single quote mode, insert an escaped single
/// quote, and then return to single quote mode.
/// Examples:
/// foo becomes 'foo'
/// foo bar becomes 'foo bar'
/// foo\ bar becomes 'foo\ bar'
/// foo's bar becomes 'foo '\''s bar'
/// foo "b"ar becomes 'foo "b"'
/// foo
/// bar becomes 'foo
/// bar'
String shellSingleQuote(String string) {
return "'${string.replaceAll("'", "'\\''")}'";
/// Like [shellSingleQuote], but if the string only contains safe ASCII
/// characters, don't quote it. Note that it's not always safe to omit the
/// quotes even if the string only has safe characters, as doing so might match
/// a shell keyword or a shell builtin in the first argument in a command. It
/// should be safe to use this for the second argument onwards in a command.
String simpleShellSingleQuote(String string) {
return RegExp(r"^[a-zA-Z0-9%+,./:_-]*$").hasMatch(string)
? string
: shellSingleQuote(string);
/// Runs a process and exits likewise if the process exits non-zero.
Future<ProcessResult> runProcess(String executable, List<String> arguments,
{bool runInShell = false}) async {
var processResult =
await, arguments, runInShell: runInShell);
if (processResult.exitCode != 0) {
var command =
[executable, ...arguments].map(simpleShellSingleQuote).join(" ");
throw Exception("Command exited ${processResult.exitCode}: $command\n"
return processResult;
/// Runs a process and exits likewise if the process exits non-zero, but let the
/// child process inherit out stdio handles.
Future<ProcessResult> runProcessInheritStdio(
String executable, List<String> arguments,
{bool runInShell = false}) async {
var process = await Process.start(executable, arguments,
mode: ProcessStartMode.inheritStdio, runInShell: runInShell);
var exitCode = await process.exitCode;
var processResult = ProcessResult(, exitCode, "", "");
if (processResult.exitCode != 0) {
var command =
[executable, ...arguments].map(simpleShellSingleQuote).join(" ");
throw Exception("Command exited ${processResult.exitCode}: $command");
return processResult;
/// Finds the branch of a builder given the list of branches.
String branchOfBuilder(String builder, List<String> branches) {
return branches.where((branch) => branch != "master").firstWhere(
(branch) => builder.endsWith("-$branch"),
orElse: () => "master");
/// Finds the named configuration to test according to the test matrix
/// information and the command line options.
Map<String, Set<Builder>> resolveNamedConfigurations(
TestMatrix testMatrix,
String requestedBranch,
List<String> requestedNamedConfigurations,
String requestedBuilder) {
assert(requestedBranch != null);
var testedConfigurations = <String, Set<Builder>>{};
var foundBuilder = false;
for (var builder in {
if (requestedBuilder != null && != requestedBuilder) {
var branch = branchOfBuilder(, testMatrix.branches);
if (branch != requestedBranch) {
if (requestedBuilder == null) {
stderr.writeln("error: Builder $requestedBuilder is on branch $branch "
"rather than $requestedBranch");
stderr.writeln("error: To compare with that branch, use: -B $branch");
return null;
foundBuilder = true;
for (var step in builder.steps.where((step) => step.isTestStep)) {
var testedConfiguration = step.testedConfiguration;
if (testedConfiguration == null) {
// This test step does not use a configuration; for example,
// because it is a simple script that does not produce results.
var namedConfiguration =;
if (requestedNamedConfigurations.isEmpty ||
requestedNamedConfigurations.contains(namedConfiguration)) {
.putIfAbsent(namedConfiguration, () => {})
if (requestedBuilder != null && !foundBuilder) {
stderr.writeln("error: Builder $requestedBuilder doesn't exist");
return null;
if (requestedBuilder != null &&
requestedNamedConfigurations.isEmpty &&
testedConfigurations.isEmpty) {
stderr.writeln("error: Builder $requestedBuilder isn't testing any named "
return null;
if (requestedNamedConfigurations.isNotEmpty) {
var hasUntestedConfiguration = false;
for (var requestedConfiguration in requestedNamedConfigurations) {
if (!testedConfigurations.containsKey(requestedConfiguration)) {
var builder = requestedBuilder != null
? "builder $requestedBuilder"
: "any builder";
stderr.writeln("error: The named configuration "
"$requestedConfiguration isn't tested on $builder");
hasUntestedConfiguration = true;
if (hasUntestedConfiguration) {
return null;
return testedConfigurations;
/// Locates the merge base between head and the [branch] on the given [remote].
Future<String> findMergeBase(String remote, String branch) async {
var arguments = ["merge-base", "$remote/$branch", "HEAD"];
var result =
await"git", arguments, runInShell: Platform.isWindows);
var stdout = result.stdout as String;
var stderr = result.stderr as String;
if (result.exitCode != 0) {
throw Exception("Failed to run: git ${arguments.join(' ')}\n"
return LineSplitter.split(stdout).first;
/// Exception thrown when looking up the build for a commit failed.
class NoResultsForCommitException implements Exception {
final String reason;
String toString() => reason;
/// The result after searching for a build of a commit.
class BuildSearchResult {
final int build;
final String commit;
BuildSearchResult(, this.commit);
/// Locates the build number of the [commit] on the [builder], or throws an
/// exception if the builder hasn't built the commit.
Future<BuildSearchResult> searchForBuild(String builder, String commit) async {
var requestUrl = Uri.parse(
var client = HttpClient();
var request = await client.getUrl(requestUrl);
var response = await request.close();
var object = await response
.transform(const Utf8Decoder())
.transform(const JsonDecoder())
.first as Map<String, dynamic>;
var builds = object["builds"] as List<dynamic>;
if (builds == null || builds.isEmpty) {
throw NoResultsForCommitException(
"Builder $builder hasn't built commit $commit");
var build = builds.last;
var tags = (build["tags"] as List).cast<String>();
var buildAddressTag =
tags.firstWhere((tag) => tag.startsWith("build_address:"));
var buildAddress = buildAddressTag.substring("build_address:".length);
var buildNumber = int.parse(buildAddress.split("/").last);
if (build["status"] != "COMPLETED") {
throw NoResultsForCommitException(
"Build $buildAddress isn't completed yet");
var resultsPath = buildFileCloudPath(builder, "$buildNumber", "results.json");
var flakyPath = buildFileCloudPath(builder, "$buildNumber", "flaky.json");
if (await lsGsutil(resultsPath) == null ||
await lsGsutil(flakyPath) == null) {
throw NoResultsForCommitException(
"Build $buildAddress did not upload results");
return BuildSearchResult(buildNumber, commit);
Future<BuildSearchResult> searchForApproximateBuild(
String builder, String commit) async {
try {
return await searchForBuild(builder, commit);
} on NoResultsForCommitException catch (e) {
print("Warning: $e, searching for an inexact previous build...");
var limit = 25;
var arguments = [
var processResult = await"git", arguments, runInShell: true);
if (processResult.exitCode != 0) {
throw Exception("Failed to list potential commits: git $arguments\n"
"exitCode: ${processResult.exitCode}\n"
"stdout: ${processResult.stdout}\n"
"stdout: ${processResult.stderr}\n");
for (var fallbackCommit
in LineSplitter.split(processResult.stdout as String)) {
try {
return await searchForBuild(builder, fallbackCommit);
} catch (e) {
"Warning: Searching for inexact baseline build: $e, continuing...");
throw NoResultsForCommitException(
"Failed to locate approximate baseline results for "
"$commit in past $limit commits");
void overrideConfiguration(Map<String, Map<String, dynamic>> results,
String configuration, String newConfiguration) {
results.forEach((String key, Map<String, dynamic> result) {
if (result["configuration"] == configuration) {
result["configuration"] = newConfiguration;
void printUsage(ArgParser parser, {String error, bool printOptions = false}) {
if (error != null) {
exitCode = 1;
Usage: test.dart -b [BUILDER] -n [CONFIGURATION] [OPTION]... [--]
Run tests and compare with the results on the given builder. Either the -n or
the -b option, or both, must be used. Any options following -- and non-option
arguments will be forwarded to invocations. The specified named
configuration's results will be downloaded from the specified builder. If only a
named configuration is specified, the results are downloaded from the
appropriate builders. If only a builder is specified, the default named
configuration is used if the builder only has a single named configuration.
Otherwise the available named configurations are listed.
See the documentation at
if (printOptions) {
} else {
print("Run test.dart --help to see all options.");
Future<void> runTests(List<String> args) async {
var parser = ArgParser();
abbr: "b", help: "Run tests like on the given builder");
abbr: "B",
help: "Select the builders building this branch",
defaultsTo: "master");
parser.addOption("commit", abbr: "C", help: "Compare with this commit");
help: "Re-run failing newly tests $deflakingCount times.");
help: "Report test failures for tests known to be flaky.\n"
"This ignores all flakiness data from CI but flakes\n"
"detected by --deflake will remain hidden");
help: "Output list of configurations.", negatable: false);
abbr: "n",
help: "The named test configuration(s) that supplies the\nvalues for all "
"test options, specifying how tests\nshould be run.");
abbr: "N",
help: "Use a different named configuration for local\ntesting than the "
"named configuration the baseline\nresults were downloaded for. The "
"results may be\ninexact if the baseline configuration is "
abbr: "R",
help: "Compare with this remote and git branch",
defaultsTo: "origin");
parser.addFlag("help", help: "Show the program usage.", negatable: false);
ArgResults options;
try {
options = parser.parse(args);
} on FormatException catch (exception) {
printUsage(parser, error: exception.message);
if (options["help"] as bool) {
printUsage(parser, printOptions: true);
if (options["list-configurations"] as bool) {
listConfigurations({"system": "all"});
var requestedBuilder = options["builder"] as String;
var requestedNamedConfigurations =
(options["named-configuration"] as List).cast<String>();
var localConfiguration = options["local-configuration"] as String;
if (requestedBuilder == null && requestedNamedConfigurations.isEmpty) {
error: "Please specify either a configuration (-n) or "
"a builder (-b)");
if (localConfiguration != null && requestedNamedConfigurations.length > 1) {
error: "Local configuration (-N) can only be used with a"
" single named configuration (-n)");
// Locate
gsutilPy =
// Load the test matrix.
var testMatrixPath = Platform.script.resolve("bots/test_matrix.json");
var testMatrix = TestMatrix.fromPath(testMatrixPath.toFilePath());
// Determine what named configuration to run and which builders to download
// existing results from.
var testedConfigurations = resolveNamedConfigurations(
options["branch"] as String,
if (testedConfigurations == null) {
// No valid configuration could be found. The error has already been
// reported by [resolveConfigurations].
exitCode = 1;
var namedConfigurations = testedConfigurations.keys.toSet().toList();
var builders =
testedConfigurations.values.expand((builders) => builders).toSet();
// Print information about the resolved builders to compare with.
for (var namedConfiguration in namedConfigurations) {
var testedBuilders = testedConfigurations[namedConfiguration];
var onWhichBuilders = testedBuilders.length == 1
? "builder ${}"
: "builders${ => "\n ${}").join()}";
if (localConfiguration != null) {
print("Testing named configuration $localConfiguration "
"compared with configuration $namedConfiguration "
"on $onWhichBuilders");
} else {
print("Testing named configuration $namedConfiguration "
"compared with $onWhichBuilders");
// Use given commit or find out where the current HEAD branched.
var commit = options["commit"] as String ??
await findMergeBase(
options["remote"] as String, options["branch"] as String);
print("Base commit is $commit");
// Store the downloaded results and our test results in a temporary directory.
var outDirectory = await Directory.systemTemp.createTemp("test.dart.");
try {
var tasks = <Future>[];
var needsConfigurationOverride = localConfiguration != null &&
localConfiguration != namedConfigurations.single;
var needsMerge = builders.length > 1;
var inexactBuilds = <String, String>{};
var previousFileName = "previous.json";
var flakyFileName = "flaky.json";
var downloadNumber = 0;
// Download the previous results and flakiness info from cloud storage.
for (var builder in builders) {
var builderName =;
if (needsMerge) {
previousFileName = "previous-$downloadNumber.json";
flakyFileName = "flaky-$downloadNumber.json";
// Use the buildbucket API to search for builds of the right commit.
print("Finding build on builder $builderName to compare with...");
var buildSearchResult =
await searchForApproximateBuild(builderName, commit);
if (buildSearchResult.commit != commit) {
print("Warning: Using commit ${buildSearchResult.commit} "
"as baseline instead of $commit for $builderName");
inexactBuilds[builderName] = buildSearchResult.commit;
var buildNumber =;
print("Downloading results from builder $builderName "
"build $buildNumber...");
buildFileCloudPath(builderName, buildNumber, "results.json"),
if (!(options["report-flakes"] as bool)) {
buildFileCloudPath(builderName, buildNumber, "flaky.json"),
// Run the tests.
var configurationsToRun = localConfiguration != null
? <String>[localConfiguration]
: namedConfigurations;
print("".padLeft(80, "="));
print("Running tests");
print("".padLeft(80, "="));
await runProcessInheritStdio(
runInShell: Platform.isWindows);
// Wait for the downloads and the test run to complete.
await Future.wait(tasks);
// Merge the results and flaky data downloaded from the builders.
var mergedResults = <String, Map<String, dynamic>>{};
var mergedFlaky = <String, Map<String, dynamic>>{};
if (needsMerge || needsConfigurationOverride) {
for (var i = 0; i < downloadNumber; ++i) {
previousFileName = needsMerge ? "previous-$i.json" : "previous.json";
var results =
await loadResultsMap("${outDirectory.path}/$previousFileName");
if (needsConfigurationOverride) {
results, namedConfigurations.single, localConfiguration);
if (!(options["report-flakes"] as bool)) {
flakyFileName = needsMerge ? "flaky-$i.json" : "flaky.json";
var flakyTests =
await loadResultsMap("${outDirectory.path}/$flakyFileName");
if (needsConfigurationOverride) {
flakyTests, namedConfigurations.single, localConfiguration);
// Write out the merged results for the builders.
if (needsMerge || needsConfigurationOverride) {
await File("${outDirectory.path}/previous.json").writeAsString( => jsonEncode(data) + "\n").join(""));
// Ensure that there is a flaky.json even if it wasn't downloaded.
if (needsMerge ||
needsConfigurationOverride ||
options["report-flakes"] as bool) {
await File("${outDirectory.path}/flaky.json").writeAsString( => jsonEncode(data) + "\n").join(""));
// Deflake results of the tests if required.
if (options["deflake"] as bool) {
await deflake(outDirectory, configurationsToRun,;
// Write out the final comparison.
print("".padLeft(80, "="));
print("Test Results");
print("".padLeft(80, "="));
var compareOutput = await runProcess(Platform.resolvedExecutable, [
if (compareOutput.stdout == "") {
print("There were no test failures.");
} else {
if (inexactBuilds.isNotEmpty) {
var builders = inexactBuilds.keys.toList()..sort();
for (var builder in builders) {
var inexactCommit = inexactBuilds[builder];
print("Warning: Results may be inexact because commit $inexactCommit "
"was used as the baseline for $builder instead of $commit");
} finally {
await outDirectory.delete(recursive: true);
Future<void> deflake(Directory outDirectory, List<String> configurations,
List<String> testPyArgs) async {
// Find the list of tests to deflake.
var deflakeListOutput = await runProcess(Platform.resolvedExecutable, [
var deflakeListPath = "${outDirectory.path}/deflake.list";
var deflakeListFile = File(deflakeListPath);
await deflakeListFile.writeAsString(deflakeListOutput.stdout as String);
// Deflake the changed tests.
var deflakingResultsPaths = <String>[];
for (var i = 1; deflakeListOutput.stdout != "" && i <= deflakingCount; i++) {
print("".padLeft(80, "="));
print("Running deflaking iteration $i");
print("".padLeft(80, "="));
var deflakeDirectory = Directory("${outDirectory.path}/$i");
await deflakeDirectory.create();
var deflakeArguments = [
await runProcessInheritStdio(
"python", ["tools/", ...deflakeArguments],
runInShell: Platform.isWindows);
// Update the flakiness information based on what we've learned.
print("Updating flakiness information...");
await runProcess(Platform.resolvedExecutable, [