blob: caa111570a09952692fc088b5c0597957c841df1 [file] [log] [blame]
// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
import 'package:smith/smith.dart';
import 'configuration.dart';
import 'path.dart';
import 'repository.dart';
import 'utils.dart';
const _defaultTestSelectors = const [
/// Specifies a single command line option.
/// The name of the specification is used as the key for the option in the Map
/// returned from the [TestOptionParser] parse method.
class _Option {
// TODO(rnystrom): Some string options use "" to mean "no value" and others
// use null. Clean that up.
_Option(, this.description,
{String abbr, List<String> values, String defaultsTo, bool hide})
: abbreviation = abbr,
values = values ?? [],
defaultValue = defaultsTo,
type = _OptionValueType.string,
verboseOnly = hide ?? false;
_Option.bool(, this.description, {String abbr, bool hide})
: abbreviation = abbr,
values = [],
defaultValue = false,
type = _OptionValueType.bool,
verboseOnly = hide ?? false;, this.description,
{String abbr, int defaultsTo, bool hide})
: abbreviation = abbr,
values = [],
defaultValue = defaultsTo,
type =,
verboseOnly = hide ?? false;
final String name;
final String description;
final String abbreviation;
final List<String> values;
final Object defaultValue;
final _OptionValueType type;
/// Only show this option in the verbose help.
final bool verboseOnly;
/// Gets the shortest command line argument used to refer to this option.
String get shortCommand => abbreviation != null ? "-$abbreviation" : command;
/// Gets the canonical long command line argument used to refer to this
/// option.
String get command => "--${name.replaceAll('_', '-')}";
enum _OptionValueType { bool, int, string }
/// Parses command line arguments and produces a test runner configuration.
class OptionsParser {
static final List<_Option> _options = [
new _Option('mode', 'Mode in which to run the tests.',
abbr: 'm', values: ['all']..addAll(Mode.names)),
new _Option(
'''How the Dart code should be compiled or statically processed.
none: Do not compile the Dart code.
precompiler: Compile into AOT snapshot before running the test.
dart2js: Compile to JavaScript using dart2js.
dart2analyzer: Perform static analysis on Dart code using the analyzer.
app_jit: Compile the Dart code into an app snapshot.
app_jitk: Compile the Dart code into Kernel and then into an app snapshot.
dartk: Compile the Dart code into Kernel before running test.
dartkb: Compile the Dart code into Kernel with bytecode before running test.
dartkp: Compile the Dart code into Kernel and then Kernel into AOT
snapshot before running the test.
spec_parser: Parse Dart code using the specification parser.''',
abbr: 'c',
values: Compiler.names),
new _Option(
'''Where the tests should be run.
vm: Run Dart code on the standalone Dart VM.
flutter: Run Dart code on the Flutter engine.
dart_precompiled: Run a precompiled snapshot on the VM without a JIT.
d8: Run JavaScript from the command line using v8.
jsshell: Run JavaScript from the command line using Firefox js-shell.
drt: Run Dart or JavaScript in the headless version of Chrome,
Content shell.
ContentShellOnAndroid: Run Dart or JavaScript in content shell on Android.
chromeOnAndroid: Run JavaScript in the specified browser.
self_check: Pass each test or its compiled output to every file under
`pkg` whose name ends with `_self_check.dart`. Each test is
given to the self_check tester as a filename on stdin using
the batch-mode protocol.
none: No runtime, compile only.''',
abbr: 'r',
values: Runtime.names),
new _Option(
'''The architecture to run tests for.
Allowed values are:
ia32, x64
arm, armv6, armv5te, arm64,
simarm, simarmv6, simarmv5te, simarm64,
simdbc, simdbc64''',
abbr: 'a',
values: ['all']..addAll(Architecture.names),
hide: true),
new _Option('system', 'The operating system to run tests on.',
abbr: 's',
values: System.names,
defaultsTo: Platform.operatingSystem,
hide: true),
new _Option(
'''The named test configuration that supplies the values for all
test options, specifying how tests should be run. Unimplemented
abbr: 'n',
hide: true),
new _Option.bool('checked', 'Run tests in checked mode.'),
new _Option.bool('strong', 'Deprecated, no-op.', hide: true),
// TODO(sigmund): rename flag once we migrate all dart2js bots to the test
// matrix.
new _Option.bool('host_checked', 'Run compiler with assertions enabled.',
hide: true),
new _Option.bool('minified', 'Enable minification in the compiler.',
hide: true),
new _Option.bool(
'csp', 'Run tests under Content Security Policy restrictions.',
hide: true),
new _Option.bool('fast_startup', 'Pass the --fast-startup flag to dart2js.',
hide: true),
new _Option.bool('fast_tests',
'Only run tests that are not marked `Slow` or `Timeout`.'),
new _Option.bool('enable_asserts',
'Pass the --enable-asserts flag to dart2js or to the vm.'),
new _Option.bool('no_preview_dart_2',
'Enable legacy Dart 1 behavior for some runtimes and compilers.',
hide: true),
new _Option.bool('use_cfe', 'Pass the --use-cfe flag to analyzer',
hide: true),
new _Option.bool('analyzer_use_fasta_parser',
'Pass the --use-fasta-parser flag to analyzer',
hide: true),
// TODO(sigmund): replace dart2js_with_kernel with preview-dart-2.
new _Option.bool(
'dart2js_with_kernel', 'Pass the --use-kernel flag to dart2js.',
hide: true),
new _Option.bool(
'dart2js_old_frontend', 'Pass the --use-old-frontend flag to dart2js.',
hide: true),
new _Option.bool('hot_reload', 'Run hot reload stress tests.', hide: true),
new _Option.bool(
'hot_reload_rollback', 'Run hot reload rollback stress tests.',
hide: true),
new _Option.bool(
'use_blobs', 'Use mmap instead of shared libraries for precompilation.',
hide: true),
new'timeout', 'Timeout in seconds.', abbr: 't'),
new _Option(
'''Progress indication mode.
Allowed values are:
compact, color, line, verbose, silent, status, buildbot, diff''',
abbr: 'p',
values: Progress.names,
new _Option('step_name', 'Step name for use by -pbuildbot.', hide: true),
new _Option.bool('report',
'Print a summary report of the number of tests, by expectation.',
hide: true),
new'tasks', 'The number of parallel tasks to run.',
abbr: 'j', defaultsTo: Platform.numberOfProcessors),
'The number of instances that the tests will be sharded over.',
defaultsTo: 1, hide: true),
'shard', 'The index of this instance when running in sharded mode.',
defaultsTo: 1, hide: true),
new _Option.bool('help', 'Print list of options.', abbr: 'h'),
new _Option.bool('verbose', 'Verbose output.', abbr: 'v'),
new _Option.bool('verify-ir', 'Verify kernel IR.', hide: true),
new _Option.bool('no-tree-shake', 'Disable kernel IR tree shaking.',
hide: true),
new _Option.bool('list', 'List tests only, do not run them.'),
new _Option.bool('list_status_files',
'List status files for test-suites. Do not run any test suites.',
hide: true),
new _Option.bool('report_in_json',
'When listing with --list, output result summary in JSON.',
hide: true),
new _Option.bool('time', 'Print timing information after running tests.'),
new _Option('dart', 'Path to dart executable.', hide: true),
new _Option('flutter', 'Path to flutter executable.', hide: true),
new _Option('drt', 'Path to content shell executable.', hide: true),
new _Option('firefox', 'Path to firefox browser executable.', hide: true),
new _Option('chrome', 'Path to chrome browser executable.', hide: true),
new _Option('safari', 'Path to safari browser executable.', hide: true),
new _Option.bool('use_sdk', '''Use compiler or runtime from the SDK.'''),
// TODO(rnystrom): This does not appear to be used. Remove?
new _Option('build_directory',
'The name of the build directory, where products are placed.',
hide: true),
new _Option('output_directory',
'The name of the output directory for storing log files.',
defaultsTo: "logs", hide: true),
new _Option.bool('noBatch', 'Do not run tests in batch mode.', hide: true),
new _Option.bool('dart2js_batch', 'Run dart2js tests in batch mode.',
hide: true),
new _Option.bool(
'append_logs', 'Do not delete old logs but rather append to them.',
hide: true),
new _Option.bool('write_debug_log',
'Don\'t write debug messages to stdout but rather to a logfile.',
hide: true),
new _Option.bool('write_test_outcome_log',
'Write test outcomes to a "${TestUtils.testOutcomeFileName}" file.',
hide: true),
new _Option.bool(
'Write test results to a "${TestUtils.resultLogFileName}" json file '
'located at the debug_output_directory.',
hide: true),
new _Option.bool(
'''Browser specific reset of configuration.
Warning: Using this option may remove your bookmarks and other
hide: true),
new _Option.bool(
'''If we see a crash that we did not expect, copy the core dumps to
hide: true),
new _Option(
'''IP address the HTTP servers should listen on. This address is also
used for browsers to connect to.''',
defaultsTo: '',
hide: true),
new'test_server_port', 'Port for test http server.',
defaultsTo: 0, hide: true),
'Port for test http server cross origin.',
defaultsTo: 0, hide: true),
new'test_driver_port', 'Port for http test driver server.',
defaultsTo: 0, hide: true),
'test_driver_error_port', 'Port for http test driver server errors.',
defaultsTo: 0, hide: true),
new _Option(
'''Machine specific options that is not captured by the regular test
options. Used to be able to make sane updates to the status files.''',
hide: true),
new _Option('vm_options', 'Extra options to send to the VM when running.',
hide: true),
new _Option(
'dart2js_options', 'Extra options for dart2js compilation step.',
hide: true),
new _Option(
'suite_dir', 'Additional directory to add to the testing matrix.',
hide: true),
new _Option('package_root', 'The package root to use for testing.',
hide: true),
new _Option('packages', 'The package spec file to use for testing.',
hide: true),
new _Option(
'''Exclude suites from default selector, only works when no selector
has been specified on the command line.''',
hide: true),
new _Option.bool(
Skip the compilation step, using the compilation artifacts left in
the output folder from a previous run. This flag will often cause
false positves and negatives, but can be useful for quick and
dirty offline testing when not making changes that affect the
hide: true),
new _Option.bool('print_passing_stdout',
'Print the stdout of passing, as well as failing, tests.',
hide: true)
/// For printing out reproducing command lines, we don't want to add these
/// options.
static final _blacklistedOptions = [
/// Parses a list of strings as test options.
/// Returns a list of configurations in which to run the tests.
/// Configurations are maps mapping from option keys to values. When
/// encountering the first non-option string, the rest of the arguments are
/// stored in the returned Map under the 'rest' key.
List<TestConfiguration> parse(List<String> arguments) {
// Help supersedes all other arguments.
if (arguments.contains("--help") || arguments.contains("-h")) {
verbose: arguments.contains("--verbose") || arguments.contains("-v"));
return null;
var configuration = <String, dynamic>{};
// Fill in configuration with arguments passed to the test script.
for (var i = 0; i < arguments.length; i++) {
var arg = arguments[i];
// Extract name and value for options.
String command;
String value;
_Option option;
if (arg.startsWith("--")) {
// A long option name.
var equals = arg.indexOf("=");
if (equals != -1) {
// A long option with a value, like "--arch=ia32".
command = arg.substring(0, equals);
value = arg.substring(equals + 1);
} else {
command = arg;
option = _findByName(command.substring(2));
} else if (arg.startsWith("-")) {
// An abbreviated option.
if (arg.length == 1) {
_fail('Missing option name after "-".');
command = arg.substring(0, 2);
if (arg.length > 2) {
// An abbreviated option followed by a value, like "-aia32".
value = arg.substring(2);
option = _findByAbbreviation(command.substring(1));
} else {
// The argument does not start with "-" or "--" and is therefore not an
// option. Use it as a test selector pattern.
var patterns = configuration.putIfAbsent("selectors", () => <String>[]);
if (option == null) {
_fail('Unknown command line option "$command".');
// If we need a value, look at the next argument.
if (value == null && option.type != _OptionValueType.bool) {
if (i + 1 >= arguments.length) {
_fail('Missing value for command line option "$command".');
value = arguments[++i];
// Multiple uses of a flag are an error, because there is no naturally
// correct way to handle conflicting options.
if (configuration.containsKey( {
_fail('Already have value for command line option "$command".');
// Parse the value for the option.
switch (option.type) {
case _OptionValueType.bool:
if (value != null) {
_fail('Boolean flag "$command" does not take a value.');
configuration[] = true;
try {
configuration[] = int.parse(value);
} on FormatException {
_fail('Integer value expected for option "$command".');
case _OptionValueType.string:
// Validate against the allowed values.
if (!option.values.isEmpty) {
for (var v in value.split(",")) {
if (!option.values.contains(v)) {
_fail('Unknown value "$v" for command line option "$command".');
// TODO(rnystrom): Store as a list instead of a comma-delimited
// string.
configuration[] = value;
// Apply default values for unspecified options.
for (var option in _options) {
if (!configuration.containsKey( {
configuration[] = option.defaultValue;
return _createConfigurations(configuration);
/// Prints [message] and exits with a non-zero exit code.
void _fail(String message) {
/// Given a set of parsed option values, returns the list of command line
/// arguments that would reproduce that configuration.
List<String> _reproducingCommand(Map<String, dynamic> data) {
var arguments = <String>[];
for (var option in _options) {
var name =;
if (!data.containsKey(name) || _blacklistedOptions.contains(name)) {
var value = data[name];
if (data[name] == option.defaultValue ||
(name == 'packages' &&
value == Repository.uri.resolve('.packages').toFilePath())) {
if (option.type != _OptionValueType.bool) {
return arguments;
List<TestConfiguration> _createConfigurations(
Map<String, dynamic> configuration) {
var selectors = _expandSelectors(configuration);
// Put observatory_ui in a configuration with its own packages override.
// Only one value in the configuration map is mutable:
if (selectors.containsKey('observatory_ui')) {
if (selectors.length == 1) {
configuration['packages'] = Repository.uri
} else {
// Make a new configuration whose selectors map only contains
// observatory_ui, and remove observatory_ui from the original
// selectors. The only mutable value in the map is the selectors, so a
// shallow copy is safe.
var observatoryConfiguration =
new Map<String, dynamic>.from(configuration);
var observatorySelectors = {
'observatory_ui': selectors['observatory_ui']
// Set the packages flag.
observatoryConfiguration['packages'] = Repository.uri
return _expandConfigurations(configuration, selectors)
observatoryConfiguration, observatorySelectors));
return _expandConfigurations(configuration, selectors);
/// Recursively expands a configuration with multiple values per key into a
/// list of configurations with exactly one value per key.
List<TestConfiguration> _expandConfigurations(
Map<String, dynamic> data, Map<String, RegExp> selectors) {
var result = <TestConfiguration>[];
// Handles a string option containing a space-separated list of words.
listOption(String name) {
var value = data[name] as String;
if (value == null) return const <String>[];
return value
.split(" ")
.map((s) => s.trim())
.where((s) => s.isNotEmpty)
var dart2jsOptions = listOption("dart2js_options");
var vmOptions = listOption("vm_options");
// JSON reporting implies listing and reporting.
if (data['report_in_json'] as bool) {
data['list'] = true;
data['report'] = true;
// Use verbose progress indication for verbose output unless buildbot
// progress indication is requested.
if ((data['verbose'] as bool) &&
(data['progress'] as String) != 'buildbot') {
data['progress'] = 'verbose';
var runtimeNames = data["runtime"] as String;
var runtimes = <Runtime>[];
if (runtimeNames != null) {
var compilerNames = data["compiler"] as String;
var compilers = <Compiler>[];
if (compilerNames != null) {
// Pick default compilers or runtimes if only one or the other is provided.
if (runtimes.isEmpty) {
if (compilers.isEmpty) {
runtimes = [Runtime.vm];
compilers = [Compiler.none];
} else {
// Pick a runtime for each compiler.
runtimes.addAll( => compiler.defaultRuntime));
} else if (compilers.isEmpty) {
// Pick a compiler for each runtime.
compilers.addAll( => runtime.defaultCompiler));
// Expand runtimes.
for (var runtime in runtimes) {
// Start installing the runtime if needed.
// Expand architectures.
var architectures = data["arch"] as String;
if (architectures == "all") {
architectures = "ia32,x64,simarm,simarm64,simdbc64";
for (var architectureName in architectures.split(",")) {
var architecture = Architecture.find(architectureName);
// Expand compilers.
for (var compiler in compilers) {
// Expand modes.
String modes = data["mode"] ??;
if (modes == "all") modes = "debug,release,product";
for (var modeName in modes.split(",")) {
var mode = Mode.find(modeName);
var system = System.find(data["system"] as String);
var namedConfiguration = getNamedConfiguration(
data["named_configuration"] as String,
var innerConfiguration = new Configuration(
namedConfiguration?.name ?? "custom configuration",
timeout: data["timeout"] as int,
enableAsserts: data["enable_asserts"] as bool,
useBlobs: data["use_blobs"] as bool,
useSdk: data["use_sdk"] as bool,
useFastStartup: data["fast_startup"] as bool,
useDart2JSWithKernel: data["dart2js_with_kernel"] as bool,
useDart2JSOldFrontEnd: data["dart2js_old_frontend"] as bool,
useHotReload: data["hot_reload"] as bool,
useHotReloadRollback: data["hot_reload_rollback"] as bool,
isChecked: data["checked"] as bool,
isHostChecked: data["host_checked"] as bool,
isCsp: data["csp"] as bool,
isMinified: data["minified"] as bool,
vmOptions: vmOptions,
builderTag: data["builder_tag"] as String,
previewDart2: !(data["no_preview_dart_2"] as bool));
var configuration = new TestConfiguration(
configuration: innerConfiguration,
namedConfiguration: namedConfiguration,
progress: Progress.find(data["progress"] as String),
selectors: selectors,
appendLogs: data["append_logs"] as bool,
batch: !(data["noBatch"] as bool),
batchDart2JS: data["dart2js_batch"] as bool,
copyCoreDumps: data["copy_coredumps"] as bool,
isVerbose: data["verbose"] as bool,
listTests: data["list"] as bool,
listStatusFiles: data["list_status_files"] as bool,
printTiming: data["time"] as bool,
printReport: data["report"] as bool,
reportInJson: data["report_in_json"] as bool,
resetBrowser: data["reset_browser_configuration"] as bool,
skipCompilation: data["skip_compilation"] as bool,
useAnalyzerCfe: data["use_cfe"] as bool,
data["analyzer_use_fasta_parser"] as bool,
useKernelBytecode: compiler == Compiler.dartkb,
writeDebugLog: data["write_debug_log"] as bool,
writeTestOutcomeLog: data["write_test_outcome_log"] as bool,
writeResultLog: data["write_result_log"] as bool,
drtPath: data["drt"] as String,
chromePath: data["chrome"] as String,
safariPath: data["safari"] as String,
firefoxPath: data["firefox"] as String,
dartPath: data["dart"] as String,
dartPrecompiledPath: data["dart_precompiled"] as String,
flutterPath: data["flutter"] as String,
taskCount: data["tasks"] as int,
shardCount: data["shards"] as int,
shard: data["shard"] as int,
stepName: data["step_name"] as String,
testServerPort: data["test_server_port"] as int,
data['test_server_cross_origin_port'] as int,
testDriverErrorPort: data["test_driver_error_port"] as int,
localIP: data["local_ip"] as String,
dart2jsOptions: dart2jsOptions,
packages: data["packages"] as String,
packageRoot: data["package_root"] as String,
suiteDirectory: data["suite_dir"] as String,
outputDirectory: data["output_directory"] as String,
reproducingArguments: _reproducingCommand(data),
fastTestsOnly: data["fast_tests"] as bool,
printPassingStdout: data["print_passing_stdout"] as bool);
if (configuration.validate()) {
return result;
/// Expands the test selectors into a suite name and a simple regular
/// expression to be used on the full path of a test file in that test suite.
/// If no selectors are explicitly given, uses the default suite patterns.
Map<String, RegExp> _expandSelectors(Map<String, dynamic> configuration) {
var selectors = configuration['selectors'];
if (selectors == null) {
if (configuration['suite_dir'] != null) {
var suitePath = new Path(configuration['suite_dir'] as String);
selectors = [suitePath.filename];
} else {
selectors = _defaultTestSelectors.toList();
var excludeSuites = configuration['exclude_suite'] != null
? configuration['exclude_suite'].split(',')
: [];
for (var exclude in excludeSuites) {
if ((selectors as List).contains(exclude)) {
} else {
print("Warning: default selectors does not contain $exclude");
var selectorMap = <String, RegExp>{};
for (var i = 0; i < (selectors as List).length; i++) {
var pattern = selectors[i] as String;
var suite = pattern;
var slashLocation = pattern.indexOf('/');
if (slashLocation != -1) {
suite = pattern.substring(0, slashLocation);
pattern = pattern.substring(slashLocation + 1);
pattern = pattern.replaceAll('*', '.*');
} else {
pattern = ".?";
if (selectorMap.containsKey(suite)) {
print("Error: '$suite/$pattern'. Only one test selection"
" pattern is allowed to start with '$suite/'");
selectorMap[suite] = new RegExp(pattern);
return selectorMap;
/// Print out usage information.
void _printHelp({bool verbose}) {
var buffer = new StringBuffer();
buffer.writeln('''The Dart SDK's internal test runner.
Usage: dart tools/test.dart [options] [selector]
The optional selector limits the tests that will be run. For example, the
selector "language/issue", or equivalently "language/*issue*", limits to test
files matching the regexp ".*issue.*\\.dart" in the "tests/language" directory.
If you specify only a runtime ("-r"), then an appropriate default compiler will
be chosen for that runtime. Likewise, if you specify only a compiler ("-c"),
then a matching runtime is chosen. If neither compiler nor runtime is selected,
the test is run directly from source on the VM.
for (var option in _options) {
if (!verbose && option.verboseOnly) continue;
if (option.abbreviation != null) {
buffer.write("-${option.abbreviation}, ");
} else {
buffer.write(" ");
switch (option.type) {
case _OptionValueType.bool:
// No value.
case _OptionValueType.string:
if (option.values.length > 6) {
// If there are many options, they won't fit nicely in one line and
// should be instead listed in the description.
} else if (option.values.isNotEmpty) {
} else {
if (option.type != _OptionValueType.bool &&
option.defaultValue != null &&
option.defaultValue != "") {
buffer.write(" (defaults to ${option.defaultValue})");
.writeln(" ${option.description.replaceAll('\n', '\n ')}");
if (!verbose) {
buffer.write('Pass "--verbose" to see more options.');
_Option _findByAbbreviation(String abbreviation) {
for (var option in _options) {
if (abbreviation == option.abbreviation) return option;
return null;
_Option _findByName(String name) {
for (var option in _options) {
if (name == return option;
// Allow hyphens instead of underscores as the separator since they are
// more common for command line flags.
if (name =="_", "-")) return option;
return null;
Configuration getNamedConfiguration(String template, Runtime runtime,
Compiler compiler, Mode mode, Architecture architecture, System system) {
if (template == null) return null;
if (template.contains(r"${runtime}")) {
template = template.replaceFirst(r"${runtime}",;
if (template.contains(r"${compiler}")) {
template = template.replaceFirst(r"${compiler}",;
if (template.contains(r"${mode}")) {
template = template.replaceFirst(r"${mode}",;
if (template.contains(r"${arch}")) {
template = template.replaceFirst(r"${arch}",;
if (template.contains(r"${system}")) {
var name = {'windows': 'win', 'macos': 'mac'}[] ??;
template = template.replaceFirst(r"${system}", name);
TestMatrix testMatrix = TestMatrix.fromPath("tools/bots/test_matrix.json");
return testMatrix.configurations
.singleWhere((c) => == template, orElse: () => null);