blob: ee648d46ee05ffb61d23f374409f1053abc020da [file] [log] [blame]
// Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:io';
import 'package:async/async.dart';
import 'package:boolean_selector/boolean_selector.dart';
import 'package:stack_trace/stack_trace.dart';
import 'package:test_api/backend.dart'
show PlatformSelector, Runtime, SuitePlatform;
import 'package:test_api/src/backend/group.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/group_entry.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/operating_system.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/suite.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/util/pretty_print.dart'; // ignore: implementation_imports
import 'runner/configuration.dart';
import 'runner/configuration/reporters.dart';
import 'runner/debugger.dart';
import 'runner/engine.dart';
import 'runner/load_exception.dart';
import 'runner/load_suite.dart';
import 'runner/loader.dart';
import 'runner/no_tests_found_exception.dart';
import 'runner/reporter.dart';
import 'runner/reporter/compact.dart';
import 'runner/reporter/expanded.dart';
import 'runner/reporter/multiplex.dart';
import 'runner/runner_suite.dart';
import 'util/io.dart';
/// A class that loads and runs tests based on a [Configuration].
/// This maintains a [Loader] and an [Engine] and passes test suites from one to
/// the other, as well as printing out tests with a [CompactReporter] or an
/// [ExpandedReporter].
class Runner {
/// The test runner configuration.
final _config = Configuration.current;
/// The loader that loads the test suites from the filesystem.
final _loader = Loader();
/// The engine that runs the test suites.
final Engine _engine;
/// The reporter that's emitting the test runner's results.
final Reporter _reporter;
/// The subscription to the stream returned by [_loadSuites].
StreamSubscription? _suiteSubscription;
/// The set of suite paths for which [_warnForUnknownTags] has already been
/// called.
/// This is used to avoid printing duplicate warnings when a suite is loaded
/// on multiple platforms.
final _tagWarningSuites = <String>{};
/// The current debug operation, if any.
/// This is stored so that we can cancel it when the runner is closed.
CancelableOperation? _debugOperation;
/// The memoizer for ensuring [close] only runs once.
final _closeMemo = AsyncMemoizer<void>();
bool get _closed => _closeMemo.hasRun;
/// Sinks created for each file reporter (if there are any).
final List<IOSink> _sinks;
/// Creates a new runner based on [configuration].
factory Runner(Configuration config) => config.asCurrent(() {
var engine = Engine(
concurrency: config.concurrency,
coverage: config.coverage,
testRandomizeOrderingSeed: config.testRandomizeOrderingSeed,
stopOnFirstFailure: config.stopOnFirstFailure,
var sinks = <IOSink>[];
Reporter createFileReporter(String reporterName, String filepath) {
final sink =
(File(filepath)..createSync(recursive: true)).openWrite();
return allReporters[reporterName]!.factory(config, engine, sink);
return Runner._(
// Standard reporter.
allReporters[config.reporter]!.factory(config, engine, stdout),
// File reporters.
for (var reporter in config.fileReporters.keys)
createFileReporter(reporter, config.fileReporters[reporter]!),
Runner._(this._engine, this._reporter, this._sinks);
/// Starts the runner.
/// This starts running tests and printing their progress. It returns whether
/// or not they ran successfully.
Future<bool> run() => _config.asCurrent(() async {
if (_closed) {
throw StateError('run() may not be called on a closed Runner.');
var suites = _loadSuites();
if (_config.coverage != null) {
await Directory(_config.coverage!).create(recursive: true);
bool? success;
if (_config.pauseAfterLoad) {
success = await _loadThenPause(suites);
} else {
var subscription =
_suiteSubscription = suites.listen(_engine.suiteSink.add);
var results = await Future.wait(<Future>[
.then((_) => _engine.suiteSink.close()),
], eagerError: true);
success = results.last as bool?;
if (_closed) return false;
if (_engine.passed.isEmpty &&
_engine.failed.isEmpty &&
_engine.skipped.isEmpty) {
if (_config.globalPatterns.isNotEmpty) {
var patterns = toSentence( =>
pattern is RegExp
? 'regular expression "${pattern.pattern}"'
: '"$pattern"'));
throw NoTestsFoundException('No tests match $patterns.');
} else if (_config.includeTags != BooleanSelector.all ||
_config.excludeTags != BooleanSelector.none) {
throw NoTestsFoundException(
'No tests match the requested tag selectors:\n'
' include: "${_config.includeTags}"\n'
' exclude: "${_config.excludeTags}"');
} else {
throw NoTestsFoundException('No tests were found.');
return (success ?? false) &&
(_engine.passed.isNotEmpty || _engine.skipped.isNotEmpty);
/// Emits a warning if the user is trying to run on a platform that's
/// unsupported for the entire package.
void _warnForUnsupportedPlatforms() {
var testOn = _config.suiteDefaults.metadata.testOn;
if (testOn == PlatformSelector.all) return;
var unsupportedRuntimes = _config.suiteDefaults.runtimes
.where((runtime) => !testOn.evaluate(currentPlatform(runtime, null)))
if (unsupportedRuntimes.isEmpty) return;
// Human-readable names for all unsupported runtimes.
var unsupportedNames = <String>[];
// If the user tried to run on one or more unsupported browsers, figure out
// whether we should warn about the individual browsers or whether all
// browsers are unsupported.
var unsupportedBrowsers =
unsupportedRuntimes.where((platform) => platform.isBrowser);
if (unsupportedBrowsers.isNotEmpty) {
var supportsAnyBrowser = _loader.allRuntimes
.where((runtime) => runtime.isBrowser)
.any((runtime) => testOn.evaluate(currentPlatform(runtime, null)));
if (supportsAnyBrowser) {
.addAll( =>;
} else {
// If the user tried to run on the VM and it's not supported, figure out if
// that's because of the current OS or whether the VM is unsupported.
if (unsupportedRuntimes.contains(Runtime.vm)) {
var supportsAnyOS = OperatingSystem.all.any((os) => testOn.evaluate(
compiler: null, os: os, inGoogle: inGoogle)));
if (supportsAnyOS) {
} else {
unsupportedNames.add('the Dart VM');
warn("this package doesn't support running tests on "
'${toSentence(unsupportedNames, conjunction: 'or')}.');
/// Closes the runner.
/// This stops any future test suites from running. It will wait for any
/// currently-running VM tests, in case they have stuff to clean up on the
/// filesystem.
Future close() => _closeMemo.runOnce(() async {
Timer? timer;
if (!_engine.isIdle) {
// Wait a bit to print this message, since printing it eagerly looks weird
// if the tests then finish immediately.
timer = Timer(const Duration(seconds: 1), () {
// Pause the reporter while we print to ensure that we don't interfere
// with its output.
print('Waiting for current test(s) to finish.');
print('Press Control-C again to terminate immediately.');
await _debugOperation?.cancel();
await _suiteSubscription?.cancel();
_suiteSubscription = null;
// Make sure we close the engine *before* the loader. Otherwise,
// LoadSuites provided by the loader may get into bad states.
// We close the loader's browsers while we're closing the engine because
// browser tests don't store any state we care about and we want them to
// shut down without waiting for their tear-downs.
await Future.wait([_loader.closeEphemeral(), _engine.close()]);
await _loader.close();
// Flush any IOSinks created for file reporters.
await Future.wait( => s.flush().then((_) => s.close())));
/// Return a stream of [LoadSuite]s in [_config.testSelections].
/// Only tests that match [_config.patterns] will be included in the
/// suites once they're loaded.
Stream<LoadSuite> _loadSuites() {
return StreamGroup.merge( {
final testPath = pathEntry.key;
final testSelections = pathEntry.value;
final suiteConfig = _config.suiteDefaults.selectTests(testSelections);
if (Directory(testPath).existsSync()) {
return _loader.loadDir(testPath, suiteConfig);
} else if (File(testPath).existsSync()) {
return _loader.loadFile(testPath, suiteConfig);
} else {
return Stream.fromIterable([
LoadException(testPath, 'Does not exist.'),
})).map((loadSuite) {
return loadSuite.changeSuite((suite) {
return _shardSuite(suite.filter((test) {
// Skip any tests that don't match all the global patterns.
if (!_config.globalPatterns
.every((pattern) => {
return false;
// If the user provided tags, skip tests that don't match all of them.
if (!_config.includeTags.evaluate(test.metadata.tags.contains)) {
return false;
// Skip tests that do match any tags the user wants to exclude.
if (_config.excludeTags.evaluate(test.metadata.tags.contains)) {
return false;
final testSelections = suite.config.testSelections;
assert(testSelections.isNotEmpty, 'Tests should have been selected');
return testSelections.any((selection) {
// Skip tests that don't match all the suite specific patterns.
if (!selection.testPatterns
.every((pattern) => {
return false;
// Skip tests that don't start on `line` or `col` if specified.
var line = selection.line;
var col = selection.col;
if (line == null && col == null) return true;
var trace = test.trace;
if (trace == null) {
throw StateError(
'Cannot filter by line/column for this test suite, no stack'
'trace available.');
var path = suite.path;
if (path == null) {
throw StateError(
'Cannot filter by line/column for this test suite, no suite'
'path available.');
// The absolute path as it will appear in stack traces.
var absoluteSuitePath = File(path).absolute.uri.toFilePath();
bool matchLineAndCol(Frame frame) {
switch (frame.uri.scheme) {
case 'file':
if (frame.uri.toFilePath() != absoluteSuitePath) {
return false;
case 'package':
// It is unlikely that the test case is specified in a
// package: URI. Ignore this case.
return false;
// Now we can assume that the kernel is compiled using
// --filesystem-scheme.
// In this case, because we don't know the --filesystem-root, as
// long as the file path matches we assume it is the same file.
if (!absoluteSuitePath.endsWith(frame.uri.path)) {
return false;
if (line != null && frame.line != line) {
return false;
if (col != null && frame.column != col) {
return false;
return true;
return trace.frames.any(matchLineAndCol);
/// Prints a warning for any unknown tags referenced in [suite] or its
/// children.
void _warnForUnknownTags(Suite suite) {
if (_tagWarningSuites.contains(suite.path)) return;
var unknownTags = _collectUnknownTags(suite);
if (unknownTags.isEmpty) return;
var yellow = _config.color ? '\u001b[33m' : '';
var bold = _config.color ? '\u001b[1m' : '';
var noColor = _config.color ? '\u001b[0m' : '';
var buffer = StringBuffer()
..write('${yellow}Warning:$noColor ')
..write(unknownTags.length == 1 ? 'A tag was ' : 'Tags were ')
..write('used that ')
..write(unknownTags.length == 1 ? "wasn't " : "weren't ")
..writeln('specified in dart_test.yaml.');
unknownTags.forEach((tag, entries) {
buffer.write(' $bold$tag$noColor was used in');
if (entries.length == 1) {
buffer.writeln(' ${_entryDescription(entries.single)}');
for (var entry in entries) {
buffer.write('\n ${_entryDescription(entry)}');
/// Collects all tags used by [suite] or its children that aren't also passed
/// on the command line.
/// This returns a map from tag names to lists of entries that use those tags.
Map<String, List<GroupEntry>> _collectUnknownTags(Suite suite) {
var unknownTags = <String, List<GroupEntry>>{};
var currentTags = <String>{};
void collect(GroupEntry entry) {
var newTags = <String>{};
for (var unknownTag
in entry.metadata.tags.difference(_config.knownTags)) {
if (currentTags.contains(unknownTag)) continue;
unknownTags.putIfAbsent(unknownTag, () => []).add(entry);
if (entry is! Group) return;
for (var child in entry.entries) {
return unknownTags;
/// Returns a human-readable description of [entry], including its type.
String _entryDescription(GroupEntry entry) {
if (entry is Test) return 'the test "${}"';
if ( return 'the group "${}"';
return 'the suite itself';
/// If sharding is enabled, filters [suite] to only include the tests that
/// should be run in this shard.
/// We just take a slice of the tests in each suite corresponding to the shard
/// index. This makes the tests pretty tests across shards, and since the
/// tests are continuous, makes us more likely to be able to re-use
/// `setUpAll()` logic.
RunnerSuite _shardSuite(RunnerSuite suite) {
if (_config.totalShards == null) return suite;
var shardSize = / _config.totalShards!;
var shardIndex = _config.shardIndex!;
var shardStart = (shardSize * shardIndex).round();
var shardEnd = (shardSize * (shardIndex + 1)).round();
var count = -1;
var filtered = suite.filter((test) {
return count >= shardStart && count < shardEnd;
return filtered;
/// Loads each suite in [suites] in order, pausing after load for runtimes
/// that support debugging.
Future<bool> _loadThenPause(Stream<LoadSuite> suites) async {
var subscription = _suiteSubscription = suites.asyncMap((loadSuite) async {
var operation = _debugOperation = debug(_engine, _reporter, loadSuite);
await operation.valueOrCancellation();
var results = await Future.wait(<Future>[
subscription.asFuture<void>().then((_) => _engine.suiteSink.close()),
], eagerError: true);
return results.last as bool;