blob: 30d903c88b3a6eaf64100a66b4e6318896869c64 [file] [log] [blame]
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
/// A generic test runner that executes a list of tests, logs test results, and
/// adds sharding support.
/// This library contains no logic related to the modular_test framework. It is
/// used to help integrate tests with our test infrastructure.
// TODO(sigmund): this library should move somewhere else.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
/// A generic test.
abstract class Test {
/// Unique test name.
String get name;
/// Run the actual test.
Future<void> run();
class RunnerOptions {
/// Name of the test suite being run.
final String suiteName;
/// Configuration name to use when writing result logs.
final String? configurationName;
/// Filter used to only run tests that match the filter name.
final String? filter;
/// Where log files are emitted.
/// Note that all shards currently emit the same filenames, so two shards
/// shouldn't be given the same [logDir] otherwise they will overwrite each
/// other's log files.
final Uri? logDir;
/// Of [shards], which shard is currently being executed.
final int shard;
/// How many shards will be used to run a suite.
final int shards;
/// Whether to print verbose information.
final bool verbose;
/// Template used to help developers reproduce the issue.
/// The following substitutions are made:
/// * %executable is replaced with `Platform.executable`
/// * %script is replaced with the current `Platform.script`
/// * %name is replaced with the test name.
final String reproTemplate;
{required this.suiteName,
required this.shard,
required this.shards,
required this.verbose,
required this.reproTemplate});
class _TestOutcome {
/// Unique test name.
final String name;
/// Whether, after running the test, the test matches its expectations.
late bool matchedExpectations;
/// Additional output emitted by the test, only used when expectations don't
/// match and more details need to be provided.
String? output;
/// Time used to run the test.
late Duration elapsedTime;
Future<void> runSuite<T>(List<Test> tests, RunnerOptions options) async {
if (options.filter == null) {
if (options.logDir == null) {
print('warning: no output directory provided, logs wont be emitted.');
if (options.configurationName == null) {
print('warning: please provide a configuration name.');
var sortedTests = tests.toList()..sort((a, b) =>;
List<_TestOutcome> testOutcomes = [];
int shard = options.shard;
int shards = options.shards;
for (int i = 0; i < sortedTests.length; i++) {
if (shards > 1 && i % shards != shard) continue;
var test = sortedTests[i];
var name =;
if (options.verbose) stdout.write('$name: ');
if (options.filter != null && !name.contains(options.filter!)) {
if (options.verbose) stdout.write('skipped\n');
var watch = Stopwatch()..start();
var outcome = _TestOutcome(;
try {
if (options.verbose) stdout.write('pass\n');
outcome.matchedExpectations = true;
} catch (e, st) {
var repro = options.reproTemplate
.replaceAll('%executable', Platform.resolvedExecutable)
.replaceAll('%script', Platform.script.path)
outcome.matchedExpectations = false;
outcome.output = 'uncaught exception: $e\n$st\nTo repro run:\n $repro';
if (options.verbose) stdout.write('fail\n${outcome.output}');
outcome.elapsedTime = watch.elapsed;
if (options.logDir == null) {
// TODO(sigmund): delete. This is only added to ensure the bots show test
// failures until support for `--output-directory` is added to the test
// matrix.
if (testOutcomes.any((o) => !o.matchedExpectations)) {
exitCode = 1;
List<String> results = [];
List<String> logs = [];
for (int i = 0; i < testOutcomes.length; i++) {
var test = testOutcomes[i];
final record = jsonEncode({
'name': '${options.suiteName}/${}',
'configuration': options.configurationName,
'suite': options.suiteName,
'time_ms': test.elapsedTime.inMilliseconds,
'expected': 'Pass',
'result': test.matchedExpectations ? 'Pass' : 'Fail',
'matches': test.matchedExpectations,
if (!test.matchedExpectations) {
final log = jsonEncode({
'name': '${options.suiteName}/${}',
'configuration': options.configurationName,
'result': test.matchedExpectations ? 'Pass' : 'Fail',
'log': test.output,
// Ensure the directory URI ends with a path separator.
var logDir = Directory.fromUri(options.logDir!).uri;
var resultJsonUri = logDir.resolve('results.json');
var logsJsonUri = logDir.resolve('logs.json');
.writeAsStringSync( => '$s\n').join(), flush: true);
.writeAsStringSync( => '$s\n').join(), flush: true);
print('log files emitted to ${resultJsonUri} and ${logsJsonUri}');