| //#!/usr/bin/env dart | |
| // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
| // for details. All rights reserved. Use of this source code is governed by a | |
| // BSD-style license that can be found in the LICENSE file. | |
| /** | |
| * testrunner is a program to run Dart unit tests. Unlike $DART/tools/test.dart, | |
| * this program is intended for 3rd parties to be able to run unit tests in | |
| * a batched fashion. As such, it adds some features and removes others. Some | |
| * of the removed features are: | |
| * | |
| * - No support for test.status files. The assumption is that tests are | |
| * expected to pass. | |
| * - A restricted set of runtimes. The assumption here is that the Dart | |
| * libraries deal with platform dependencies, and so the primary | |
| * SKUs that a user of this app would be concerned with would be | |
| * Dart-native versus compiled, and client (browser) vs server. To | |
| * support these, three runtimes are allowed: 'drt-dart' and 'drt-js' (for | |
| * client native and client-compiled, respectively), and 'vm' | |
| * (for server-side native). | |
| * - No sharding of test processes. | |
| * | |
| * On the other hand, a number of features have been added: | |
| * | |
| * - The ability to filter tests by group or name. | |
| * - The ability to run tests in isolates. | |
| * - The ability to customize the format of the test result messages. | |
| * - The ability to list the tests available. | |
| * | |
| * By default, testrunner will run all tests in the current directory. | |
| * With a -R option, it will recurse into subdirectories. | |
| * Directories can also be specified on the command line; if | |
| * any are specified they will override the use of the current directory. | |
| * All files that match the `--test-file-pattern` will be included; by default | |
| * this is files with names that end in _test.dart. | |
| * | |
| * Options can be specified on the command line, via a configuration | |
| * file (`--config`) or via a test.config file in the test directory, | |
| * in decreasing order of priority. | |
| * | |
| * The three runtimes are: | |
| * | |
| * vm - run native Dart in the VM; i.e. using $DARTSDK/dart-sdk/bin/dart. | |
| * drt-dart - run native Dart in DumpRenderTree, the headless version of | |
| * Dartium, which is located in $DARTSDK/chromium/DumpRenderTree, if | |
| * you intsalled the SDK that is bundled with the editor, or available | |
| * from http://gsdview.appspot.com/dartium-archive/continuous/ | |
| * otherwise. | |
| * | |
| * drt-js - run Dart compiled to Javascript in DumpRenderTree. | |
| * | |
| * testrunner supports simple DOM render tests. These can use expected values | |
| * for the render output from DumpRenderTree, either are textual DOM | |
| * descriptions (`--layout-tests`) or pixel renderings (`--pixel-tests`). | |
| * When running layout tests, testrunner will see if there is a file with | |
| * a .png or a .txt extension in a directory with the same name as the | |
| * test file (without extension) and with the test name as the file name. | |
| * For example, if there is a test file foo_test.dart with tests 'test1' | |
| * and 'test2', it will look for foo_test/test1.txt and foo_test/test2.txt | |
| * for text render layout files. If these exist it will do additional checks | |
| * of the rendered layout; if not, the test will fail. | |
| * | |
| * Layout file (re)generation can be done using `--regenerate`. This will | |
| * create or update the layout files (and implicitly pass the tests). | |
| * | |
| * The wrapping and execution of test files is handled by test_pipeline.dart, | |
| * which is run in an isolate. The `--pipeline` argument can be used to | |
| * specify a different script for running a test file pipeline, allowing | |
| * customization of the pipeline. | |
| */ | |
| // TODO - layout tests that use PNGs rather than DRT text render dumps. | |
| library testrunner; | |
| import 'dart:io'; | |
| import 'dart:isolate'; | |
| import 'dart:math'; | |
| import 'package:args/args.dart'; | |
| part 'options.dart'; | |
| part 'utils.dart'; | |
| /** The set of [PipelineRunner]s to execute. */ | |
| List _tasks; | |
| /** The maximum number of pipelines that can run concurrently. */ | |
| int _maxTasks; | |
| /** The number of pipelines currently running. */ | |
| int _numTasks; | |
| /** The index of the next pipeline runner to execute. */ | |
| int _nextTask; | |
| /** The stream to use for high-value messages, like test results. */ | |
| OutputStream _outStream; | |
| /** The stream to use for low-value messages, like verbose output. */ | |
| OutputStream _logStream; | |
| /** | |
| * The user can specify output streams on the command line, using 'none', | |
| * 'stdout', 'stderr', or a file path; [getStream] will take such a name | |
| * and return an appropriate [OutputStream]. | |
| */ | |
| OutputStream getStream(String name) { | |
| if (name == null || name == 'none') { | |
| return null; | |
| } | |
| if (name == 'stdout') { | |
| return stdout; | |
| } | |
| if (name == 'stderr') { | |
| return stderr; | |
| } | |
| return new File(name).openOutputStream(FileMode.WRITE); | |
| } | |
| /** | |
| * Given a [List] of [testFiles], either print the list or create | |
| * and execute pipelines for the files. | |
| */ | |
| void processTests(Map config, List testFiles) { | |
| _outStream = getStream(config['out']); | |
| _logStream = getStream(config['log']); | |
| if (config['list-files']) { | |
| if (_outStream != null) { | |
| for (var i = 0; i < testFiles.length; i++) { | |
| _outStream.writeString(testFiles[i]); | |
| _outStream.writeString('\n'); | |
| } | |
| } | |
| } else { | |
| _maxTasks = min(config['tasks'], testFiles.length); | |
| _numTasks = 0; | |
| _nextTask = 0; | |
| spawnTasks(config, testFiles); | |
| } | |
| } | |
| /** Execute as many tasks as possible up to the maxTasks limit. */ | |
| void spawnTasks(Map config, List testFiles) { | |
| var verbose = config['verbose']; | |
| // If we were running in the VM and the immediate flag was set, we have | |
| // already printed the important messages (i.e. prefixed with ###), | |
| // so we should skip them now. | |
| var skipNonVerbose = config['immediate'] && config['runtime'] == 'vm'; | |
| while (_numTasks < _maxTasks && _nextTask < testFiles.length) { | |
| ++_numTasks; | |
| var testfile = testFiles[_nextTask++]; | |
| config['testfile'] = testfile; | |
| ReceivePort port = new ReceivePort(); | |
| port.receive((msg, _) { | |
| List stdout = msg[0]; | |
| List stderr = msg[1]; | |
| List log = msg[2]; | |
| int exitCode = msg[3]; | |
| writelog(stdout, _outStream, _logStream, verbose, skipNonVerbose); | |
| writelog(stderr, _outStream, _logStream, true, skipNonVerbose); | |
| writelog(log, _outStream, _logStream, verbose, skipNonVerbose); | |
| port.close(); | |
| --_numTasks; | |
| if (exitCode == 0 || !config['stopOnFailure']) { | |
| spawnTasks(config, testFiles); | |
| } | |
| if (_numTasks == 0) { | |
| // No outstanding tasks; we're all done. | |
| // We could later print a summary report here. | |
| } | |
| }); | |
| SendPort s = spawnUri(config['pipeline']); | |
| s.send(config, port.toSendPort()); | |
| } | |
| } | |
| /** | |
| * Our tests are configured so that critical messages have a '###' prefix. | |
| * [writeLog] takes the output from a pipeline execution and writes it to | |
| * our output streams. It will strip the '###' if necessary on critical | |
| * messages; other messages will only be written if verbose output was | |
| * specified. | |
| */ | |
| void writelog(List messages, OutputStream out, OutputStream log, | |
| bool includeVerbose, bool skipNonVerbose) { | |
| for (var i = 0; i < messages.length; i++) { | |
| var msg = messages[i]; | |
| if (msg.startsWith('###')) { | |
| if (!skipNonVerbose && out != null) { | |
| out.writeString(msg.substring(3)); | |
| out.writeString('\n'); | |
| } | |
| } else if (msg.startsWith('CONSOLE MESSAGE:')) { | |
| if (!skipNonVerbose && out != null) { | |
| int idx = msg.indexOf('###'); | |
| if (idx > 0) { | |
| out.writeString(msg.substring(idx + 3)); | |
| out.writeString('\n'); | |
| } | |
| } | |
| } else if (includeVerbose && log != null) { | |
| log.writeString(msg); | |
| log.writeString('\n'); | |
| } | |
| } | |
| } | |
| sanitizeConfig(Map config, ArgParser parser) { | |
| config['layout'] = config['layout-text'] || config['layout-pixel']; | |
| // TODO - check if next three are actually used. | |
| config['runInBrowser'] = (config['runtime'] != 'vm'); | |
| config['verbose'] = (config['log'] != 'none' && !config['list-groups']); | |
| config['filtering'] = (config['include'].length > 0 || | |
| config['exclude'].length > 0); | |
| config['timeout'] = int.parse(config['timeout']); | |
| config['tasks'] = int.parse(config['tasks']); | |
| var dartsdk = config['dartsdk']; | |
| var pathSep = Platform.pathSeparator; | |
| if (dartsdk != null) { | |
| if (parser.getDefault('dart2js') == config['dart2js']) { | |
| config['dart2js'] = | |
| '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}dart2js'; | |
| } | |
| if (parser.getDefault('dart') == config['dart']) { | |
| config['dart'] = '$dartsdk${pathSep}dart-sdk${pathSep}bin${pathSep}dart'; | |
| } | |
| if (parser.getDefault('drt') == config['drt']) { | |
| config['drt'] = '$dartsdk${pathSep}chromium${pathSep}DumpRenderTree'; | |
| } | |
| } | |
| config['unittest'] = makePathAbsolute(config['unittest']); | |
| config['drt'] = makePathAbsolute(config['drt']); | |
| config['dart'] = makePathAbsolute(config['dart']); | |
| config['dart2js'] = makePathAbsolute(config['dart2js']); | |
| config['runnerDir'] = runnerDirectory; | |
| } | |
| main() { | |
| var optionsParser = getOptionParser(); | |
| var options = loadConfiguration(optionsParser); | |
| if (isSane(options)) { | |
| if (options['list-options']) { | |
| printOptions(optionsParser, options, false, stdout); | |
| } else if (options['list-all-options']) { | |
| printOptions(optionsParser, options, true, stdout); | |
| } else { | |
| var config = new Map(); | |
| for (var option in options.options) { | |
| config[option] = options[option]; | |
| } | |
| var rest = []; | |
| // Process the remmaining command line args. If they look like | |
| // options then split them up and add them to the map; they may be for | |
| // custom pipelines. | |
| for (var other in options.rest) { | |
| var idx; | |
| if (other.startsWith('--') && (idx = other.indexOf('=')) > 0) { | |
| var optName = other.substring(2, idx); | |
| var optValue = other.substring(idx+1); | |
| config[optName] = optValue; | |
| } else { | |
| rest.add(other); | |
| } | |
| } | |
| sanitizeConfig(config, optionsParser); | |
| // Build the list of tests and then execute them. | |
| List dirs = rest; | |
| if (dirs.length == 0) { | |
| dirs.add('.'); // Use current working directory as default. | |
| } | |
| buildFileList(dirs, | |
| new RegExp(options['test-file-pattern']), options['recurse'], | |
| (f) => processTests(config, f)); | |
| } | |
| } | |
| } |