pkg/test_runner/lib/test_runner.dart

// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

// Run tests like on the given builder and/or named configuration.

import 'dart:async';
import 'dart:convert';
import 'dart:io';

import 'package:args/args.dart';
import 'package:smith/smith.dart';

import 'bot_results.dart';
import 'src/options.dart';

const int deflakingCount = 5;

/// Quotes a string in shell single quote mode. This function produces a single
/// shell argument that evaluates to the exact string provided, handling any
/// special characters in the input string. Shell single quote mode works uses
/// the single quote character as the delimiter and uses the characters
/// in-between verbatim without any special processing. To insert the single
/// quote character itself, escape single quote mode, insert an escaped single
/// quote, and then return to single quote mode.
///
/// Examples:
///   foo becomes 'foo'
///   foo bar becomes 'foo bar'
///   foo\ bar becomes 'foo\ bar'
///   foo's bar becomes 'foo '\''s bar'
///   foo "b"ar becomes 'foo "b"'
///   foo
///   bar becomes 'foo
///   bar'
String shellSingleQuote(String string) {
  return "'${string.replaceAll("'", r"'\''")}'";
}

/// Like [shellSingleQuote], but if the string only contains safe ASCII
/// characters, don't quote it. Note that it's not always safe to omit the
/// quotes even if the string only has safe characters, as doing so might match
/// a shell keyword or a shell builtin in the first argument in a command. It
/// should be safe to use this for the second argument onwards in a command.
String simpleShellSingleQuote(String string) {
  return RegExp(r"^[a-zA-Z\d%+,./:_\-]*$").hasMatch(string)
      ? string
      : shellSingleQuote(string);
}

/// Runs a process and exits likewise if the process exits non-zero.
Future<ProcessResult> runProcess(
  String executable,
  List<String> arguments, {
  bool runInShell = false,
}) async {
  var processResult = await Process.run(
    executable,
    arguments,
    runInShell: runInShell,
  );
  if (processResult.exitCode != 0) {
    var command = [
      executable,
      ...arguments,
    ].map(simpleShellSingleQuote).join(" ");
    throw Exception(
      "Command exited ${processResult.exitCode}: $command\n"
      "${processResult.stdout}\n${processResult.stderr}",
    );
  }
  return processResult;
}

/// Runs a process and exits likewise if the process exits non-zero, but let the
/// child process inherit out stdio handles.
Future<ProcessResult> runProcessInheritStdio(
  String executable,
  List<String> arguments, {
  bool runInShell = false,
}) async {
  var process = await Process.start(
    executable,
    arguments,
    mode: ProcessStartMode.inheritStdio,
    runInShell: runInShell,
  );
  var exitCode = await process.exitCode;
  var processResult = ProcessResult(process.pid, exitCode, "", "");
  if (processResult.exitCode != 0) {
    var command = [
      executable,
      ...arguments,
    ].map(simpleShellSingleQuote).join(" ");
    throw Exception("Command exited ${processResult.exitCode}: $command");
  }
  return processResult;
}

/// Finds the branch of a builder given the list of branches.
String branchOfBuilder(String builder, List<String> branches) {
  return branches
      .where((branch) => branch != "main")
      .firstWhere(
        (branch) => builder.endsWith("-$branch"),
        orElse: () => "main",
      );
}

/// Finds the named configuration to test according to the test matrix
/// information and the command line options.
Map<String, Set<Builder>>? resolveNamedConfigurations(
  TestMatrix testMatrix,
  String requestedBranch,
  List<String> requestedNamedConfigurations,
  String? requestedBuilder,
) {
  var testedConfigurations = <String, Set<Builder>>{};
  var foundBuilder = false;
  for (var builder in testMatrix.builders) {
    if (requestedBuilder != null && builder.name != requestedBuilder) {
      continue;
    }
    var branch = branchOfBuilder(builder.name, testMatrix.branches);
    if (branch != requestedBranch) {
      if (requestedBuilder == null) {
        continue;
      }
      stderr.writeln(
        "error: Builder $requestedBuilder is on branch $branch "
        "rather than $requestedBranch",
      );
      stderr.writeln("error: To compare with that branch, use: -B $branch");
      return null;
    }
    foundBuilder = true;
    for (var step in builder.steps.where((step) => step.isTestStep)) {
      var testedConfiguration = step.testedConfiguration;
      if (testedConfiguration == null) {
        // This test step does not use a configuration; for example,
        // because it is a simple script that does not produce results.
        continue;
      }
      var namedConfiguration = testedConfiguration.name;
      if (requestedNamedConfigurations.isEmpty ||
          requestedNamedConfigurations.contains(namedConfiguration)) {
        testedConfigurations
            .putIfAbsent(namedConfiguration, () => {})
            .add(builder);
      }
    }
  }
  if (requestedBuilder != null && !foundBuilder) {
    stderr.writeln("error: Builder $requestedBuilder doesn't exist");
    return null;
  }
  if (requestedBuilder != null &&
      requestedNamedConfigurations.isEmpty &&
      testedConfigurations.isEmpty) {
    stderr.writeln(
      "error: Builder $requestedBuilder isn't testing any named "
      "configurations",
    );
    return null;
  }
  if (requestedNamedConfigurations.isNotEmpty) {
    var hasUntestedConfiguration = false;
    for (var requestedConfiguration in requestedNamedConfigurations) {
      if (!testedConfigurations.containsKey(requestedConfiguration)) {
        var builder = requestedBuilder != null
            ? "builder $requestedBuilder"
            : "any builder";
        stderr.writeln(
          "error: The named configuration "
          "$requestedConfiguration isn't tested on $builder",
        );
        hasUntestedConfiguration = true;
      }
    }
    if (hasUntestedConfiguration) {
      return null;
    }
  }

  return testedConfigurations;
}

/// Locates the merge base between head and the [branch] on the given [remote].
Future<String> findMergeBase(String? remote, String? branch) async {
  var arguments = ["merge-base", "$remote/$branch", "HEAD"];
  var result = await Process.run(
    "git",
    arguments,
    runInShell: Platform.isWindows,
  );

  var stdout = result.stdout as String;
  var stderr = result.stderr as String;

  if (result.exitCode != 0) {
    throw Exception(
      "Failed to run: git ${arguments.join(' ')}\n"
      "stdout:\n$stdout\n"
      "stderr:\n$stderr\n",
    );
  }
  return LineSplitter.split(stdout).first;
}

/// Exception thrown when looking up the build for a commit failed.
class NoResultsForCommitException implements Exception {
  final String reason;

  NoResultsForCommitException(this.reason);

  @override
  String toString() => reason;
}

/// The result after searching for a build of a commit.
class BuildSearchResult {
  final int build;
  final String commit;

  BuildSearchResult(this.build, this.commit);
}

/// Locates the build number of the [commit] on the [builder], or throws an
/// exception if the builder hasn't built the commit.
Future<BuildSearchResult> searchForBuild(String builder, String commit) async {
  var requestBody = jsonEncode({
    "fields": "builds.*.status,builds.*.number",
    "predicate": {
      "builder": {
        "project": "dart",
        "bucket": "ci.sandbox",
        "builder": builder,
      },
      "tags": [
        {
          "key": "buildset",
          "value": "commit/gitiles/dart.googlesource.com/sdk/+/$commit",
        },
      ],
    },
  });
  var requestUrl = Uri.https(
    "cr-buildbucket.appspot.com",
    "prpc/buildbucket.v2.Builds/SearchBuilds",
  );
  var client = HttpClient();
  var request = await client.postUrl(requestUrl);
  request.headers.add(HttpHeaders.acceptHeader, ContentType.json.mimeType);
  request.headers.add(HttpHeaders.contentTypeHeader, ContentType.json.mimeType);
  request.write(requestBody);
  var response = await request.close();
  var responseString = await const Utf8Decoder().bind(response).join();
  client.close();
  // Remove XSSI protection prefix )]}'\n before parsing the response.
  var object = jsonDecode(responseString.substring(5)) as Map<String, dynamic>;
  var builds = object["builds"] as List<dynamic>?;
  if (builds == null || builds.isEmpty) {
    throw NoResultsForCommitException(
      "Builder $builder hasn't built commit $commit",
    );
  }
  var build = builds.last;
  var buildNumber = build["number"] as int;
  if (!{"SUCCESS", "FAILURE"}.contains(build["status"])) {
    throw NoResultsForCommitException("Build $buildNumber isn't completed yet");
  }
  var resultsPath = buildFileCloudPath(builder, "$buildNumber", "results.json");
  var flakyPath = buildFileCloudPath(builder, "$buildNumber", "flaky.json");
  if ((await lsGsutil(resultsPath)).isEmpty ||
      (await lsGsutil(flakyPath)).isEmpty) {
    throw NoResultsForCommitException(
      "Build $buildNumber did not upload results",
    );
  }
  return BuildSearchResult(buildNumber, commit);
}

Future<BuildSearchResult> searchForApproximateBuild(
  String builder,
  String commit,
) async {
  try {
    return await searchForBuild(builder, commit);
  } on NoResultsForCommitException catch (e) {
    print("Warning: $e, searching for an inexact previous build...");
    var limit = 25;
    var arguments = [
      "rev-list",
      "$commit~$limit..$commit~1",
      "--first-parent",
      "--topo-order",
    ];
    var processResult = await Process.run("git", arguments, runInShell: true);
    if (processResult.exitCode != 0) {
      throw Exception(
        "Failed to list potential commits: git $arguments\n"
        "exitCode: ${processResult.exitCode}\n"
        "stdout: ${processResult.stdout}\n"
        "stdout: ${processResult.stderr}\n",
      );
    }
    for (var fallbackCommit in LineSplitter.split(
      processResult.stdout as String,
    )) {
      try {
        return await searchForBuild(builder, fallbackCommit);
      } catch (e) {
        print(
          "Warning: Searching for inexact baseline build: $e, continuing...",
        );
      }
    }
    throw NoResultsForCommitException(
      "Failed to locate approximate baseline results for "
      "$commit in past $limit commits",
    );
  }
}

void overrideConfiguration(
  Map<String, Map<String, dynamic>> results,
  String configuration,
  String? newConfiguration,
) {
  results.forEach((String key, Map<String, dynamic> result) {
    if (result["configuration"] == configuration) {
      result["configuration"] = newConfiguration;
    }
  });
}

void printUsage(ArgParser parser, {String? error, bool printOptions = false}) {
  if (error != null) {
    print("$error\n");
    exitCode = 1;
  }
  print("""
Usage: test.dart -b [BUILDER] -n [CONFIGURATION] [OPTION]... [--]
                 [TEST.PY OPTION]... [SELECTOR]...

Run tests and compare with the results on the given builder. Either the -n or
the -b option, or both, must be used. Any options following -- and non-option
arguments will be forwarded to test.py invocations. The specified named
configuration's results will be downloaded from the specified builder. If only a
named configuration is specified, the results are downloaded from the
appropriate builders. If only a builder is specified, the default named
configuration is used if the builder only has a single named configuration.
Otherwise the available named configurations are listed.

See the documentation at https://goto.google.com/dart-status-file-free-workflow
""");
  if (printOptions) {
    print(parser.usage);
  } else {
    print("Run test.dart --help to see all options.");
  }
}

Future<void> runTests(List<String> args) async {
  var parser = ArgParser();
  parser.addOption(
    "builder",
    abbr: "b",
    help: "Run tests like on the given builder",
  );
  parser.addOption(
    "branch",
    abbr: "B",
    help: "Select the builders building this branch",
    defaultsTo: "main",
  );
  parser.addOption("commit", abbr: "C", help: "Compare with this commit");
  parser.addFlag(
    "deflake",
    help: "Re-run failing newly tests $deflakingCount times.",
  );
  parser.addFlag(
    "report-flakes",
    help:
        "Report test failures for tests known to be flaky.\n"
        "This ignores all flakiness data from CI but flakes\n"
        "detected by --deflake will remain hidden",
  );
  parser.addFlag(
    "list-configurations",
    help: "Output list of configurations.",
    negatable: false,
  );
  parser.addMultiOption(
    "named-configuration",
    abbr: "n",
    help:
        "The named test configuration(s) that supplies the\nvalues for all "
        "test options, specifying how tests\nshould be run.",
  );
  parser.addOption(
    "local-configuration",
    abbr: "N",
    help:
        "Use a different named configuration for local\ntesting than the "
        "named configuration the baseline\nresults were downloaded for. The "
        "results may be\ninexact if the baseline configuration is "
        "different.",
  );
  parser.addOption(
    "remote",
    abbr: "R",
    help: "Compare with this remote and git branch",
    defaultsTo: "origin",
  );
  parser.addFlag("help", help: "Show the program usage.", negatable: false);

  ArgResults options;
  try {
    options = parser.parse(args);
  } on FormatException catch (exception) {
    printUsage(parser, error: exception.message);
    return;
  }

  if (options["help"] as bool) {
    printUsage(parser, printOptions: true);
    return;
  }

  if (options["list-configurations"] as bool) {
    listConfigurations({"system": "all"});
    return;
  }

  var requestedBuilder = options["builder"] as String?;
  var requestedNamedConfigurations = (options["named-configuration"] as List)
      .cast<String>();
  var localConfiguration = options["local-configuration"] as String?;

  if (requestedBuilder == null && requestedNamedConfigurations.isEmpty) {
    printUsage(
      parser,
      error:
          "Please specify either a configuration (-n) or "
          "a builder (-b)",
    );
    return;
  }

  if (localConfiguration != null && requestedNamedConfigurations.length > 1) {
    printUsage(
      parser,
      error:
          "Local configuration (-N) can only be used with a"
          " single named configuration (-n)",
    );
    return;
  }

  // Locate gsutil.py.
  gsutilPy = Platform.script
      .resolve("../third_party/gsutil/gsutil.py")
      .toFilePath();

  // Load the test matrix.
  var testMatrixPath = Platform.script.resolve("bots/test_matrix.json");
  var testMatrix = TestMatrix.fromPath(testMatrixPath.toFilePath());
  // Determine what named configuration to run and which builders to download
  // existing results from.
  var testedConfigurations = resolveNamedConfigurations(
    testMatrix,
    options["branch"] as String,
    requestedNamedConfigurations,
    requestedBuilder,
  );
  if (testedConfigurations == null) {
    // No valid configuration could be found. The error has already been
    // reported by [resolveConfigurations].
    exitCode = 1;
    return;
  }
  var namedConfigurations = testedConfigurations.keys.toSet().toList();
  var builders = testedConfigurations.values
      .expand((builders) => builders)
      .toSet();

  // Print information about the resolved builders to compare with.
  for (var namedConfiguration in namedConfigurations) {
    var testedBuilders = testedConfigurations[namedConfiguration]!;
    var onWhichBuilders = testedBuilders.length == 1
        ? "builder ${testedBuilders.single.name}"
        : "builders${testedBuilders.map((b) => "\n  ${b.name}").join()}";
    if (localConfiguration != null) {
      print(
        "Testing named configuration $localConfiguration "
        "compared with configuration $namedConfiguration "
        "on $onWhichBuilders",
      );
    } else {
      print(
        "Testing named configuration $namedConfiguration "
        "compared with $onWhichBuilders",
      );
    }
  }

  // Use given commit or find out where the current HEAD branched.
  var commit =
      options["commit"] as String? ??
      await findMergeBase(
        options["remote"] as String?,
        options["branch"] as String?,
      );
  print("Base commit is $commit");

  // Store the downloaded results and our test results in a temporary directory.
  var outDirectory = await Directory.systemTemp.createTemp("test.dart.");
  try {
    var tasks = <Future>[];
    var needsConfigurationOverride =
        localConfiguration != null &&
        localConfiguration != namedConfigurations.single;
    var needsMerge = builders.length > 1;
    var inexactBuilds = <String, String>{};
    var previousFileName = "previous.json";
    var flakyFileName = "flaky.json";
    var downloadNumber = needsMerge ? 0 : 1;

    // Download the previous results and flakiness info from cloud storage.
    for (var builder in builders) {
      var builderName = builder.name;
      if (needsMerge) {
        previousFileName = "previous-$downloadNumber.json";
        flakyFileName = "flaky-$downloadNumber.json";
        downloadNumber++;
      }

      // Use the buildbucket API to search for builds of the right commit.
      print("Finding build on builder $builderName to compare with...");
      var buildSearchResult = await searchForApproximateBuild(
        builderName,
        commit,
      );
      if (buildSearchResult.commit != commit) {
        print(
          "Warning: Using commit ${buildSearchResult.commit} "
          "as baseline instead of $commit for $builderName",
        );
        inexactBuilds[builderName] = buildSearchResult.commit;
      }

      var buildNumber = buildSearchResult.build.toString();
      print(
        "Downloading results from builder $builderName "
        "build $buildNumber...",
      );
      tasks.add(
        cpGsutil(
          buildFileCloudPath(builderName, buildNumber, "results.json"),
          "${outDirectory.path}/$previousFileName",
        ),
      );
      if (!(options["report-flakes"] as bool)) {
        tasks.add(
          cpGsutil(
            buildFileCloudPath(builderName, buildNumber, "flaky.json"),
            "${outDirectory.path}/$flakyFileName",
          ),
        );
      }
    }

    // Run the tests.
    var configurationsToRun = localConfiguration != null
        ? <String>[localConfiguration]
        : namedConfigurations;
    print("".padLeft(80, "="));
    print("Running tests");
    print("".padLeft(80, "="));
    await runProcessInheritStdio("python3", [
      "tools/test.py",
      "--named-configuration=${configurationsToRun.join(",")}",
      "--output-directory=${outDirectory.path}",
      "--clean-exit",
      "--silent-failures",
      "--write-results",
      "--write-logs",
      ...options.rest,
    ], runInShell: Platform.isWindows);
    // Wait for the downloads and the test run to complete.
    await Future.wait(tasks);
    // Merge the results and flaky data downloaded from the builders.
    var mergedResults = <String, Map<String, dynamic>>{};
    var mergedFlaky = <String, Map<String, dynamic>>{};
    if (needsMerge || needsConfigurationOverride) {
      for (var i = 0; i < downloadNumber; ++i) {
        previousFileName = needsMerge ? "previous-$i.json" : "previous.json";
        var results = await loadResultsMap(
          "${outDirectory.path}/$previousFileName",
        );
        if (needsConfigurationOverride) {
          overrideConfiguration(
            results,
            namedConfigurations.single,
            localConfiguration,
          );
        }
        mergedResults.addAll(results);
        if (!(options["report-flakes"] as bool)) {
          flakyFileName = needsMerge ? "flaky-$i.json" : "flaky.json";
          var flakyTests = await loadResultsMap(
            "${outDirectory.path}/$flakyFileName",
          );
          if (needsConfigurationOverride) {
            overrideConfiguration(
              flakyTests,
              namedConfigurations.single,
              localConfiguration,
            );
          }
          mergedFlaky.addAll(flakyTests);
        }
      }
    }
    // Write out the merged results for the builders.
    if (needsMerge || needsConfigurationOverride) {
      await File("${outDirectory.path}/previous.json").writeAsString(
        mergedResults.values.map((data) => "${jsonEncode(data)}\n").join(""),
      );
    }
    // Ensure that there is a flaky.json even if it wasn't downloaded.
    if (needsMerge ||
        needsConfigurationOverride ||
        options["report-flakes"] as bool) {
      await File("${outDirectory.path}/flaky.json").writeAsString(
        mergedFlaky.values.map((data) => "${jsonEncode(data)}\n").join(""),
      );
    }
    // Deflake results of the tests if required.
    if (options["deflake"] as bool) {
      await deflake(outDirectory, configurationsToRun, options.rest);
    }
    // Write out the final comparison.
    print("".padLeft(80, "="));
    print("Test Results");
    print("".padLeft(80, "="));
    var compareOutput = await runProcess(Platform.resolvedExecutable, [
      "tools/bots/compare_results.dart",
      "--human",
      "--verbose",
      "--changed",
      "--failing",
      "--passing",
      "--flakiness-data=${outDirectory.path}/flaky.json",
      "--logs=${outDirectory.path}/logs.json",
      "${outDirectory.path}/previous.json",
      "${outDirectory.path}/results.json",
    ]);
    if (compareOutput.stdout == "") {
      print("No new unapproved test failures detected.");
    } else {
      stdout.write(compareOutput.stdout);
    }
    if (inexactBuilds.isNotEmpty) {
      print("");
      var builders = inexactBuilds.keys.toList()..sort();
      for (var builder in builders) {
        var inexactCommit = inexactBuilds[builder];
        print(
          "Warning: Results may be inexact because commit $inexactCommit "
          "was used as the baseline for $builder instead of $commit",
        );
      }
    }
  } finally {
    await outDirectory.delete(recursive: true);
  }
}

Future<void> deflake(
  Directory outDirectory,
  List<String> configurations,
  List<String> testPyArgs,
) async {
  // Find the list of tests to deflake.
  var deflakeListOutput = await runProcess(Platform.resolvedExecutable, [
    "tools/bots/compare_results.dart",
    "--name-only",
    "--changed",
    "--failing",
    "--passing",
    "--flakiness-data=${outDirectory.path}/flaky.json",
    "${outDirectory.path}/previous.json",
    "${outDirectory.path}/results.json",
  ]);
  var deflakeListPath = "${outDirectory.path}/deflake.list";
  var deflakeListFile = File(deflakeListPath);
  await deflakeListFile.writeAsString(deflakeListOutput.stdout as String);

  // Deflake the changed tests.
  var deflakingResultsPaths = <String>[];
  for (var i = 1; deflakeListOutput.stdout != "" && i <= deflakingCount; i++) {
    print("".padLeft(80, "="));
    print("Running deflaking iteration $i");
    print("".padLeft(80, "="));
    var deflakeDirectory = Directory("${outDirectory.path}/$i");
    await deflakeDirectory.create();
    var deflakeArguments = [
      "--named-configuration=${configurations.join(",")}",
      "--output-directory=${deflakeDirectory.path}",
      "--clean-exit",
      "--silent-failures",
      "--write-results",
      "--test-list=$deflakeListPath",
      ...testPyArgs,
    ];

    await runProcessInheritStdio("python3", [
      "tools/test.py",
      ...deflakeArguments,
    ], runInShell: Platform.isWindows);
    deflakingResultsPaths.add("${deflakeDirectory.path}/results.json");
  }

  // Update the flakiness information based on what we've learned.
  print("Updating flakiness information...");
  await runProcess(Platform.resolvedExecutable, [
    "tools/bots/update_flakiness.dart",
    "--input=${outDirectory.path}/flaky.json",
    "--output=${outDirectory.path}/flaky.json",
    "${outDirectory.path}/results.json",
    ...deflakingResultsPaths,
  ]);
}