blob: 3d5714f158cc6dd5364bf270083695eb65f19506 [file] [log] [blame]
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
part of testrunner;
/** Create and return an options parser for the test runner. */
ArgParser getOptionParser() {
var parser = new ArgParser();
parser.addOption('help', abbr: '?',
help: 'Show usage information.');
parser.addOption('runtime', abbr: 'r', defaultsTo: 'vm',
help: 'Where the tests should be run.',
allowed: ['vm', 'drt-dart', 'drt-js'],
allowedHelp: {
'vm': 'Run Dart code natively on the standalone dart vm.',
// TODO(antonm): fix option name.
'drt-dart': 'Run Dart code natively in the headless version of\n'
'Chrome, Content shell.',
// TODO(antonm): fix option name.
'drt-js': 'Run Dart compiled to JavaScript in the headless version\n'
'of Chrome, Content shell.'
parser.addFlag('checked', defaultsTo: false,
help: 'Run tests in checked mode.');
parser.addFlag('sort', defaultsTo: false,
help: 'Sort test files before running.');
parser.addFlag('layout-text', defaultsTo: false,
help: 'Run text layout tests.');
parser.addFlag('layout-pixel', defaultsTo: false,
help: 'Run pixel layout tests.');
parser.addOption('timeout', abbr: 't',
help: 'Timeout in seconds', defaultsTo: '60');
parser.addOption('tasks', abbr: 'j',
defaultsTo: Platform.numberOfProcessors.toString(),
help: 'The number of parallel tasks to run.');
parser.addOption('out', abbr: 'o', defaultsTo: 'stdout',
help: 'File to send test results. This should be a '
'file name or one of stdout, stderr, or none.');
help: 'Format for test list result output.');
help: 'Format for passing test result output.');
help: 'Format for failed test result output.');
help: 'Format for tests with errors result output.');
parser.addFlag('summary', defaultsTo: false,
help: 'Print a summary of tests passed/failed for each test file.');
parser.addOption('log', abbr: 'l', defaultsTo: 'none',
help: 'File to send test log/print output to. This should be a '
'file name or one of stdout, stderr, or none.');
// TODO(gram) - add loglevel once we have switched unittest to use the log
// library.
parser.addFlag('list-files', defaultsTo: false,
help: 'List test files only, do not run them.');
parser.addFlag('list-tests', defaultsTo: false,
help: 'List tests only, do not run them.');
parser.addFlag('list-groups', defaultsTo: false,
help: 'List test groups only, do not run tests.');
parser.addFlag('clean-files', defaultsTo: false,
help: 'Remove the generated files from the temporary directory.');
parser.addFlag('list-options', defaultsTo: false,
help: 'Print non-default option settings, usable as a test.config.');
parser.addFlag('list-all-options', defaultsTo: false,
help: 'Print all option settings, usable as a test.config.');
help: 'Print timing information after running tests',
defaultsTo: false);
parser.addFlag('stop-on-failure', defaultsTo: false,
help: 'Stop execution after first file with failures.');
parser.addFlag('isolate', defaultsTo: false,
help: 'Runs each test in a separate isolate.');
parser.addOption('configfile', help: 'Path to an argument file to load.');
parser.addOption('dartsdk', help: 'Path to dart SDK.');
var tmp;
if (Platform.operatingSystem == 'windows') {
tmp = "c:\\tmp\\test";
} else {
tmp = "/tmp/test";
parser.addOption('tempdir', help: 'Directory to store temp files.',
defaultsTo: tmp);
help: 'A regular expression that test file names must match '
'to be considered', defaultsTo: '_test.dart\$');
help: 'Only run tests from the specified group(s).',
allowMultiple: true);
help: 'Exclude tests from the specified group(s).',
allowMultiple: true);
parser.addFlag('recurse', abbr: 'R',
help: 'Recurse through child directories looking for tests.',
defaultsTo: false);
help: 'Print test results immediately, instead of at the end of a test '
'file. Note that in some async cases this may result in multiple '
'messages for a single test.',
defaultsTo: false);
help: 'Regenerate layout test expectation files.',
defaultsTo: false);
parser.addFlag('server', help: 'Run an HTTP server.', defaultsTo: false);
parser.addOption('port', help: 'Port to use for HTTP server');
help: 'Root directory for HTTP server for static files');
help: 'Pipeline script to use to run each test file.',
defaultsTo: 'run_pipeline.dart');
return parser;
/** Print a value option, quoting it if it has embedded spaces. */
_printValueOption(String name, value, IOSink dest) {
if (value.indexOf(' ') >= 0) {
} else {
/** Print the current option values. */
printOptions(ArgParser parser, ArgResults arguments,
bool includeDefaults, IOSink dest) {
if (dest == null) return;
for (var name in arguments.options) {
if (!name.startsWith('list-')) {
var value = arguments[name];
var defaultValue = parser.getDefault(name);
if (value is bool) {
if (includeDefaults || (value != defaultValue)) {
dest.write('--${value ? "" : "no-"}$name\n');
} else if (value is List) {
if (value.length > 0) {
for (var v in value) {
_printValueOption(name, v, dest);
} else if (value != null && (includeDefaults || value != defaultValue)) {
_printValueOption(name, value, dest);
* Get the test runner configuration. This loads options from multiple
* sources, in increasing order of priority: a test.config file in the
* current directory, a test config file specified with --configfile on
* the command line, and other arguments specified on the command line.
ArgResults loadConfiguration(optionsParser) {
var options = new List();
// We first load options from a test.config file in the working directory.
options.addAll(getFileContents('test.config', false).
where((e) => e.trim().length > 0 && e[0] != '#'));
// Next we look to see if the command line included a -testconfig argument,
// and if so, load options from that file too; where these are not
// multi-valued they will take precedence over the ones in test.config.
var commandLineArgs = new Options().arguments;
var cfgarg = '--configfile';
var cfgarge = '--configfile=';
for (var i = 0; i < commandLineArgs.length; i++) {
if (commandLineArgs[i].startsWith(cfgarg)) {
if (commandLineArgs[i] == cfgarg) {
if (i == commandLineArgs.length - 1) {
throw new Exception('Missing argument to $cfgarg');
options.addAll(getFileContents(commandLineArgs[++i], true).
where((e) => e.trim().length > 0 && e[0] != '#'));
} else if (commandLineArgs[i].startsWith(cfgarge)) {
getFileContents(commandLineArgs[i].substring(cfgarge.length), true).
where((e) => e.trim().length > 0 && e[0] != '#'));
} else {
throw new Exception('Missing argument to $cfgarg');
// Finally, we add options from the command line. These have the highest
// precedence of all.
// Now try parse the whole collection of options, and if this fails,
// issue a usage message.
try {
return optionsParser.parse(options);
} catch (e) {
print('Usage: testrunner <options> [<directory or file> ...]');
return null;
/** Perform some sanity checking of the configuration. */
bool isSane(ArgResults config) {
if (config == null) {
return false;
if (config['runtime'] == null) {
print('Missing required option --runtime');
return false;
if (config['include'].length > 0 &&
config['exclude'].length > 0) {
print('--include and --exclude are mutually exclusive.');
return false;
if ((config['layout-text'] || config['layout-pixel']) &&
config['runtime'] == 'vm') {
print('Layout tests must use --runtime values of "drt-dart" or "drt-js"');
return false;
return true;