blob: fac1224913b9da792267b48920186fadb6c71b0b [file] [log] [blame]
// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:io';
import 'dart:math';
import 'package:args/args.dart';
import 'package:boolean_selector/boolean_selector.dart';
import 'package:test_api/backend.dart';
import 'package:test_api/scaffolding.dart' show Timeout;
import '../../util/io.dart';
import '../compiler_selection.dart';
import '../configuration.dart';
import '../runtime_selection.dart';
import 'reporters.dart';
import 'values.dart';
/// The parser used to parse the command-line arguments.
final ArgParser _parser = (() {
var parser = ArgParser(allowTrailingOptions: true);
var allRuntimes = Runtime.builtIn.toList()..remove(Runtime.vm);
if (!Platform.isMacOS) allRuntimes.remove(Runtime.safari);
if (!Platform.isWindows) allRuntimes.remove(Runtime.internetExplorer);
abbr: 'h', negatable: false, help: 'Show this usage information.');
negatable: false, help: 'Show the package:test version.');
// Note that defaultsTo declarations here are only for documentation purposes.
// We pass null instead of the default so that it merges properly with the
// config file.
parser.addSeparator('Selecting Tests:');
abbr: 'n',
help: 'A substring of the name of the test to run.\n'
'Regular expression syntax is supported.\n'
'If passed multiple times, tests must match all substrings.',
splitCommas: false);
abbr: 'N',
help: 'A plain-text substring of the name of the test to run.\n'
'If passed multiple times, tests must match all substrings.',
splitCommas: false);
abbr: 't',
help: 'Run only tests with all of the specified tags.\n'
'Supports boolean selector syntax.');
parser.addMultiOption('tag', hide: true);
abbr: 'x',
help: "Don't run tests with any of the specified tags.\n"
'Supports boolean selector syntax.');
parser.addMultiOption('exclude-tag', hide: true);
help: 'Run skipped tests instead of skipping them.');
parser.addSeparator('Running Tests:');
// The UI term "platform" corresponds with the implementation term "runtime".
// The [Runtime] class used to be called [TestPlatform], but it was changed to
// avoid conflicting with [SuitePlatform]. We decided not to also change the
// UI to avoid a painful migration.
abbr: 'p',
help: 'The platform(s) on which to run the tests.\n'
'[vm (default), '
'${ => runtime.identifier).join(", ")}].\n'
'Each platform supports the following compilers:\n'
'${ => r.supportedCompilersText).join('\n')}');
abbr: 'c',
help: 'The compiler(s) to use to run tests, supported compilers are '
'[${ => c.identifier).join(', ')}].\n'
'Each platform has a default compiler but may support other '
'You can target a compiler to a specific platform using arguments '
'of the following form [<platform-selector>:]<compiler>.\n'
'If a platform is specified but no given compiler is supported for '
'that platform, then it will use its default compiler.');
abbr: 'P', help: 'The configuration preset(s) to use.');
abbr: 'j',
help: 'The number of concurrent test suites run.',
defaultsTo: defaultConcurrency.toString(),
valueHelp: 'threads');
help: 'The total number of invocations of the test runner being run.');
help: 'The index of this test runner invocation (of --total-shards).');
help: 'The port of a pub serve instance serving "test/".',
valueHelp: 'port');
help: 'The default test timeout. For example: 15s, 2x, none',
defaultsTo: '30s');
help: 'Ignore all timeouts (useful if debugging)', negatable: false);
help: 'Pause for debugging before any tests execute.\n'
'Implies --concurrency=1, --debug, and --ignore-timeouts.\n'
'Currently only supported for browser tests.',
negatable: false);
help: 'Run the VM and Chrome tests in debug mode.', negatable: false);
help: 'Gather coverage and output it to the specified directory.\n'
'Implies --debug.',
valueHelp: 'directory');
help: 'Use chained stack traces to provide greater exception details\n'
'especially for asynchronous code. It may be useful to disable\n'
'to provide improved test performance but at the cost of\n'
defaultsTo: false);
help: "Don't rerun tests that have retry set.",
defaultsTo: false,
negatable: false);
help: '**DEPRECATED**: This is now just an alias for --compiler source.',
defaultsTo: false,
hide: true,
negatable: false);
help: 'Use the specified seed to randomize the execution order of test'
' cases.\n'
'Must be a 32bit unsigned integer or "random".\n'
'If "random", pick a random seed to use.\n'
'If not passed, do not randomize test case execution order.');
help: 'Stop running tests after the first failure.\n');
var reporterDescriptions = <String, String>{
for (final MapEntry(:key, :value) in allReporters.entries)
if (!value.hidden) key: value.description
abbr: 'r',
help: 'Set how to print test results.',
defaultsTo: defaultReporter,
allowed: allReporters.keys,
allowedHelp: reporterDescriptions,
valueHelp: 'option');
help: 'Enable an additional reporter writing test results to a file.\n'
'Should be in the form <reporter>:<filepath>, '
'Example: "json:reports/tests.json"');
negatable: false, help: 'Emit stack traces with core library frames.');
negatable: false,
help: 'Emit raw JavaScript stack traces for browser tests.');
help: 'Use terminal colors.\n(auto-detected by default)');
/// The following options are used only by the internal Google test runner.
/// They're hidden and not supported as stable API surface outside Google.
help: 'The path to the configuration file.', hide: true);
help: 'Extra arguments to pass to dart2js.', hide: true);
// If we're running test/dir/my_test.dart, we'll look for
// test/dir/my_test.dart.html in the precompiled directory.
help: 'The path to a mirror of the package directory containing HTML '
'that points to precompiled JS.',
hide: true);
return parser;
/// The usage string for the command-line arguments.
String get usage => _parser.usage;
/// Parses the configuration from [args].
/// Throws a [FormatException] if [args] are invalid.
Configuration parse(List<String> args) => _Parser(args).parse();
void _parseTestSelection(
String option, Map<String, Set<TestSelection>> selections) {
if (Platform.isWindows) {
// If given a path that starts with what looks like a drive letter, convert it
// into a file scheme URI. We can't parse using `Uri.file` because we do
// support query parameters which aren't valid file uris.
if (option.indexOf(':') == 1) {
option = 'file:///$option';
final uri = Uri.parse(option);
// Decode the path segment. Specifically, on github actions back slashes on
// windows end up being encoded into the URI instead of converted into forward
// slashes.
var path = Uri.decodeComponent(uri.path);
// Strip out the leading slash before the drive letter on windows.
if (Platform.isWindows &&
path.startsWith('/') &&
path.length >= 3 &&
path[2] == ':') {
path = path.substring(1);
final names = uri.queryParametersAll['name'];
final fullName = uri.queryParameters['full-name'];
final line = uri.queryParameters['line'];
final col = uri.queryParameters['col'];
if (names != null && names.isNotEmpty && fullName != null) {
throw const FormatException(
'Cannot specify both "name=<...>" and "full-name=<...>".',
final selection = TestSelection(
testPatterns: fullName != null
? {RegExp('^${RegExp.escape(fullName)}\$')}
: {
if (names != null)
for (var name in names) RegExp(name)
line: line == null ? null : int.parse(line),
col: col == null ? null : int.parse(col),
selections.update(path, (selections) => selections..add(selection),
ifAbsent: () => {selection});
/// A class for parsing an argument list.
/// This is used to provide access to the arg results across helper methods.
class _Parser {
/// The parsed options.
final ArgResults _options;
_Parser(List<String> args) : _options = _parser.parse(args);
List<String> _readMulti(String name) => _options[name] as List<String>;
/// Returns the parsed configuration.
Configuration parse() {
var patterns = [
for (var value in _readMulti('name'))
_wrapFormatException(value, () => RegExp(value), optionName: 'name'),
var includeTags = {..._readMulti('tags'), ..._readMulti('tag')}
.fold<BooleanSelector>(BooleanSelector.all, (selector, tag) {
return selector.intersection(BooleanSelector.parse(tag));
var excludeTags = {
}.fold<BooleanSelector>(BooleanSelector.none, (selector, tag) {
return selector.union(BooleanSelector.parse(tag));
var shardIndex = _parseOption('shard-index', int.parse);
var totalShards = _parseOption('total-shards', int.parse);
if ((shardIndex == null) != (totalShards == null)) {
throw const FormatException(
'--shard-index and --total-shards may only be passed together.');
} else if (shardIndex != null) {
if (shardIndex < 0) {
throw const FormatException('--shard-index may not be negative.');
} else if (shardIndex >= totalShards!) {
throw const FormatException(
'--shard-index must be less than --total-shards.');
var reporter = _ifParsed('reporter') as String?;
var testRandomizeOrderingSeed =
_parseOption('test-randomize-ordering-seed', (value) {
var seed = value == 'random'
? Random().nextInt(4294967295)
: int.parse(value).toUnsigned(32);
// TODO(#1547): Less hacky way of not breaking the json reporter
if (reporter != 'json') {
print('Shuffling test order with --test-randomize-ordering-seed=$seed');
return seed;
var color = _ifParsed<bool>('color') ?? canUseSpecialChars;
var runtimes =
var compilerSelections = _ifParsed<List<String>>('compiler')
if (_ifParsed<bool>('use-data-isolate-strategy') == true) {
compilerSelections ??= [];
final paths = ? null :;
Map<String, Set<TestSelection>>? selections;
if (paths != null) {
selections = {};
for (final path in paths) {
_parseTestSelection(path, selections);
return Configuration(
help: _ifParsed('help'),
version: _ifParsed('version'),
verboseTrace: _ifParsed('verbose-trace'),
chainStackTraces: _ifParsed('chain-stack-traces'),
jsTrace: _ifParsed('js-trace'),
pauseAfterLoad: _ifParsed('pause-after-load'),
debug: _ifParsed('debug'),
color: color,
configurationPath: _ifParsed('configuration'),
dart2jsArgs: _ifParsed('dart2js-args'),
precompiledPath: _ifParsed('precompiled'),
reporter: reporter,
fileReporters: _parseFileReporterOption(),
coverage: _ifParsed('coverage'),
pubServePort: _parseOption('pub-serve', int.parse),
concurrency: _parseOption('concurrency', int.parse),
shardIndex: shardIndex,
totalShards: totalShards,
timeout: _parseOption('timeout', Timeout.parse),
globalPatterns: patterns,
compilerSelections: compilerSelections,
runtimes: runtimes,
runSkipped: _ifParsed('run-skipped'),
chosenPresets: _ifParsed('preset'),
testSelections: selections,
includeTags: includeTags,
excludeTags: excludeTags,
noRetry: _ifParsed('no-retry'),
testRandomizeOrderingSeed: testRandomizeOrderingSeed,
ignoreTimeouts: _ifParsed('ignore-timeouts'),
stopOnFirstFailure: _ifParsed('fail-fast'),
// Config that isn't supported on the command line
addTags: null,
allowTestRandomization: null,
allowDuplicateTestNames: null,
customHtmlTemplatePath: null,
defineRuntimes: null,
filename: null,
foldTraceExcept: null,
foldTraceOnly: null,
onPlatform: null,
overrideRuntimes: null,
presets: null,
retry: null,
skip: null,
skipReason: null,
testOn: null,
tags: null);
/// Returns the parsed option for [name], or `null` if none was parsed.
/// If the user hasn't explicitly chosen a value, we want to pass null values
/// to [] so that it considers those fields unset when
/// merging with configuration from the config file.
T? _ifParsed<T>(String name) =>
_options.wasParsed(name) ? _options[name] as T : null;
/// Runs [parse] on the value of the option [name], and wraps any
/// [FormatException] it throws with additional information.
T? _parseOption<T>(String name, T Function(String) parse) {
if (!_options.wasParsed(name)) return null;
var value = _options[name];
if (value == null) return null;
return _wrapFormatException(value, () => parse(value as String),
optionName: name);
Map<String, String>? _parseFileReporterOption() =>
_parseOption('file-reporter', (value) {
if (!value.contains(':')) {
throw const FormatException(
'option must be in the form <reporter>:<filepath>, e.g. '
final sep = value.indexOf(':');
final reporter = value.substring(0, sep);
if (!allReporters.containsKey(reporter)) {
throw FormatException('"$reporter" is not a supported reporter');
return {reporter: value.substring(sep + 1)};
/// Runs [parse], and wraps any [FormatException] it throws with additional
/// information.
T _wrapFormatException<T>(Object? value, T Function() parse,
{String? optionName}) {
try {
return parse();
} on FormatException catch (error) {
throw FormatException(
'Couldn\'t parse ${optionName == null ? '' : '--$optionName '}"$value": '
extension _RuntimeDescription on Runtime {
String get supportedCompilersText {
var message = StringBuffer('[$identifier]: ');
message.write('${defaultCompiler.identifier} (default)');
for (var compiler in supportedCompilers) {
if (compiler == defaultCompiler) continue;
message.write(', ${compiler.identifier}');
return message.toString();