tools/migration/bin/run_tests.dart

// Copyright (c) 2017, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

/// Runs the tests in a batch on the various configurations used on the bots.

import 'dart:async';
import 'dart:io';

import 'package:args/args.dart';
import 'package:path/path.dart' as p;

import 'package:migration/src/fork.dart';
import 'package:migration/src/log.dart';

const appJit = "--compiler=app_jit";
const dart2js = "--compiler=dart2js";
const dartdevc = "--compiler=dartdevc";
const dartdevk = "--compiler=dartdevk";
const noCompiler = "--compiler=none";
const precompiler = "--compiler=precompiler";
const analyzer = "--compiler=dart2analyzer";
const dartk = "--compiler=dartk";
const dartkp = "--compiler=dartkp";

const chrome = "--runtime=chrome";
const precompiled = "--runtime=dart_precompiled";
const noRuntime = "--runtime=none";
const vm = "--runtime=vm";
const d8 = "--runtime=d8";
const jsshell = "--runtime=jsshell";

const checked = "--checked";
const dart2jsBatch = "--dart2js-batch";
const dart2jsWithKernel = "--dart2js-with-kernel";
const fastStartup = "--fast-startup";
const useSdk = "--use-sdk";
const releaseMode = "--mode=release";
const productMode = "--mode=product";
const strong = "--strong";
const previewDart2 = "--preview-dart-2";

/// Maps configuration names to a corresponding set of test.dart command line
/// arguments.
///
/// Each configuration name starts with the name of a column on the buildbot
/// waterfall (except for "dartjs-linux" which is just called "dart2js" here)
/// possibly followed by some modifier for a specific bot or annotated step on
/// a bot. The configs here are ordered the same order as the waterfall.
final allConfigs = {
  "vm-checked": [noCompiler, vm, checked],
  "vm-kernel-strong": [dartk, releaseMode, vm, strong],
  "vm-kernel-precomp-strong": [dartkp, releaseMode, precompiled, strong],
  "vm-precomp-checked": [precompiler, precompiled, checked],
  // TODO(rnystrom): Add dart2js-d8-hostchecked, dart2js-d8-minified, or
  // dart2js-jsshell?
  "analyzer": [analyzer, noRuntime, useSdk],
  "analyzer-checked": [analyzer, noRuntime, checked, useSdk],
  "analyzer-checked-strong": [analyzer, noRuntime, checked, strong, useSdk],
  "analyzer-strong": [analyzer, noRuntime, strong, useSdk],
  "dart2js": [dart2js, chrome, useSdk, dart2jsBatch],
  "dart2js-d8-checked": [
    dart2js,
    d8,
    checked,
    fastStartup,
    useSdk,
    dart2jsBatch
  ],
  "dart2js-d8-withkernel": [
    dart2js,
    d8,
    dart2jsWithKernel,
    useSdk,
    dart2jsBatch
  ],
  "dart2js-jsshell": [dart2js, jsshell, fastStartup, useSdk, dart2jsBatch],
  // TODO(rnystrom): Is it worth running dart2js on Firefox too?
  "dart2js-2": [dart2js, chrome, dart2jsBatch, previewDart2],
  "dartdevc": [dartdevc, chrome, useSdk, strong],
  "dartdevc-kernel": [dartdevk, chrome, checked, useSdk, strong],
  "dartdevc-kernel-noruntime": [dartdevk, noRuntime, checked, useSdk, strong],
};

/// A subset of the configurations that run quickly and give a decent amount of
/// coverage for the platforms that do not implement Dart 2.0 yet.
final oneConfigs = const [
  "vm",
  "vm-checked",
  "analyzer",
  "analyzer-checked",
  "dart2js",
  "dart2js-d8-checked",
  "dartdevc",
];

/// The configurations that should correctly implement Dart 2.0 (more or less)
/// already.
final twoConfigs = const [
  "analyzer-checked-strong",
  "analyzer-strong",
  "dartdevc"
];

final buildSteps = [
  // The SDK, which also builds the VM.
  ["--mode=release", "create_sdk"],
  // The kernel service.
  ["--mode=release", "kernel-service"],
  // Precompiled runtime for release
  ["--mode=release", "runtime_precompiled"],
  // Product version of the runtime and precompiled runtime.
  ["--mode=product", "runtime", "runtime_precompiled"],
  // Dartdevc and its dependencies.
  ["--mode=release", "dartdevc_test"],
];

Future<Null> main(List<String> arguments) async {
  var argParser = new ArgParser(allowTrailingOptions: true);
  argParser.addFlag("build", help: "Build runtimes before running tests.");
  argParser.addOption("config",
      abbr: "c", allowMultiple: true, help: "Which configurations to run.");
  argParser.addFlag("help");
  argParser.addFlag("1",
      abbr: "1", help: "Run some of the 1.0-supporting configurations.");
  argParser.addFlag("2",
      abbr: "2", help: "Run the 2.0-supporting configurations.");

  var argResults = argParser.parse(arguments);
  if (argResults["help"] as bool) {
    usage(argParser);
    return;
  }

  String start;
  String end;

  if (argResults.rest.length == 1) {
    // Just run a single test.
    start = argResults.rest[0];
    end = start;
  } else if (argResults.rest.length == 2) {
    start = argResults.rest[0];
    end = argResults.rest[1];
  } else {
    usage(argParser);
    exit(1);
  }

  var build = argResults["build"] as bool;
  var configs = argResults["config"] as List<String>;

  if (argResults["1"] as bool) {
    configs.addAll(oneConfigs);
  }

  if (argResults["2"] as bool) {
    configs.addAll(twoConfigs);
  }

  if (configs.isEmpty) configs.addAll(allConfigs.keys);

  var tests = scanTests();

  var startIndex = findFork(tests, start);
  var endIndex = findFork(tests, end);

  if (startIndex == null || endIndex == null) exit(1);

  tests = tests.sublist(startIndex, endIndex + 1);

  if (tests.isEmpty) {
    print("No tests in range.");
    return;
  }

  // Build any needed targets first.
  if (build) {
    for (var steps in buildSteps) {
      var command = "tools/build.py ${steps.join(' ')}";
      print("Building ${bold(command)}:");
      var exitCode = await run("tools/build.py", steps);
      if (exitCode != 0) {
        print(red("Build failed: $command"));
      }
    }
  }

  // Splits the tests into selectors and patterns.
  var selectors = <String, List<String>>{};
  for (var test in tests) {
    var parts = p.split(p.withoutExtension(test.twoPath));
    var selector = parts[0];
    var path = parts.skip(1).join("/");
    selectors.putIfAbsent(selector, () => []).add(path);
  }

  var failed = <String>[];
  var passed = <String>[];
  for (var name in configs) {
    var configArgs = allConfigs[name];
    print("${bold(name)} ${configArgs.join(' ')}:");

    var args = ["--progress=diff"];

    args.addAll(configArgs);

    if (!args.any((arg) => arg.startsWith("--mode"))) {
      args.add("--mode=release");
    }

    selectors.forEach((selector, paths) {
      args.add("$selector/${paths.join('|')}");
    });

    var exitCode = await run("tools/test.py", args);
    if (exitCode != 0) {
      print(red("Configuration failed: $name"));
      failed.add(name);
    } else {
      passed.add(name);
    }

    print("");
  }

  if (failed.length == 0) {
    var s = passed.length == 1 ? "" : "s";
    print("${green('PASSED')} all ${bold(passed.length)} configuration$s!");
  } else {
    if (passed.length > 0) {
      var s = passed == 1 ? "" : "s";
      print("${green('PASSED')} ${bold(passed.length)} configuration$s:");
      for (var config in passed) {
        print("- ${bold(config)}");
      }
    }

    var s = failed == 1 ? "" : "s";
    print("${red("FAILED")} ${bold(failed.length)} configuration$s:");
    for (var config in failed) {
      print("- ${bold(config)}");
    }
  }
}

void usage(ArgParser parser) {
  print("Usage: dart run_tests.dart [--build] [-2] [-1] [--configs=...]"
      "<first file> [last file]");
  print("\n");
  print("Example:");
  print("\n");
  print("    \$ dart run_tests.dart map_to_string queue");
  print("\n");
  print(parser.usage);
}

Future<int> run(String executable, List<String> arguments) async {
  var process = await Process.start(executable, arguments);
  process.stdout.listen((bytes) {
    stdout.add(bytes);
  });

  process.stderr.listen((bytes) {
    stderr.add(bytes);
  });

  return await process.exitCode;
}