blob: fe2a85f5a103ad05a0bd9d010d862429fe28762d [file] [log] [blame]
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
/// A generic test runner that executes a list of tests, logs test results, and
/// adds sharding support.
///
/// This library contains no logic related to the modular_test framework. It is
/// used to help integrate tests with our test infrastructure.
// TODO(sigmund): this library should move somewhere else.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
/// A generic test.
abstract class Test {
/// Unique test name.
String get name;
/// Run the actual test.
Future<void> run();
}
class RunnerOptions {
/// Name of the test suite being run.
final String suiteName;
/// Configuration name to use when writing result logs.
final String? configurationName;
/// Filter used to only run tests that match the filter name.
final String? filter;
/// Where log files are emitted.
///
/// Note that all shards currently emit the same filenames, so two shards
/// shouldn't be given the same [logDir] otherwise they will overwrite each
/// other's log files.
final Uri? logDir;
/// Of [shards], which shard is currently being executed.
final int shard;
/// How many shards will be used to run a suite.
final int shards;
/// Whether to print verbose information.
final bool verbose;
/// Template used to help developers reproduce the issue.
///
/// The following substitutions are made:
/// * %executable is replaced with `Platform.executable`
/// * %script is replaced with the current `Platform.script`
/// * %name is replaced with the test name.
final String reproTemplate;
RunnerOptions(
{required this.suiteName,
this.configurationName,
this.filter,
this.logDir,
required this.shard,
required this.shards,
required this.verbose,
required this.reproTemplate});
}
class _TestOutcome {
/// Unique test name.
final String name;
/// Whether, after running the test, the test matches its expectations.
late bool matchedExpectations;
/// Additional output emitted by the test, only used when expectations don't
/// match and more details need to be provided.
String? output;
/// Time used to run the test.
late Duration elapsedTime;
_TestOutcome(this.name);
}
Future<void> runSuite<T>(List<Test> tests, RunnerOptions options) async {
if (options.filter == null) {
if (options.logDir == null) {
print('warning: no output directory provided, logs wont be emitted.');
}
if (options.configurationName == null) {
print('warning: please provide a configuration name.');
}
}
var sortedTests = tests.toList()..sort((a, b) => a.name.compareTo(b.name));
List<_TestOutcome> testOutcomes = [];
int shard = options.shard;
int shards = options.shards;
for (int i = 0; i < sortedTests.length; i++) {
if (shards > 1 && i % shards != shard) continue;
var test = sortedTests[i];
var name = test.name;
if (options.verbose) stdout.write('$name: ');
if (options.filter != null && !name.contains(options.filter!)) {
if (options.verbose) stdout.write('skipped\n');
continue;
}
var watch = new Stopwatch()..start();
var outcome = new _TestOutcome(test.name);
try {
await test.run();
if (options.verbose) stdout.write('pass\n');
outcome.matchedExpectations = true;
} catch (e, st) {
var repro = options.reproTemplate
.replaceAll('%executable', Platform.resolvedExecutable)
.replaceAll('%script', Platform.script.path)
.replaceAll('%name', test.name);
outcome.matchedExpectations = false;
outcome.output = 'uncaught exception: $e\n$st\nTo repro run:\n $repro';
if (options.verbose) stdout.write('fail\n${outcome.output}');
}
watch.stop();
outcome.elapsedTime = watch.elapsed;
testOutcomes.add(outcome);
}
if (options.logDir == null) {
// TODO(sigmund): delete. This is only added to ensure the bots show test
// failures until support for `--output-directory` is added to the test
// matrix.
if (testOutcomes.any((o) => !o.matchedExpectations)) {
exitCode = 1;
}
return;
}
List<String> results = [];
List<String> logs = [];
for (int i = 0; i < testOutcomes.length; i++) {
var test = testOutcomes[i];
final record = jsonEncode({
'name': '${options.suiteName}/${test.name}',
'configuration': options.configurationName,
'suite': options.suiteName,
'test_name': test.name,
'time_ms': test.elapsedTime.inMilliseconds,
'expected': 'Pass',
'result': test.matchedExpectations ? 'Pass' : 'Fail',
'matches': test.matchedExpectations,
});
results.add(record);
if (!test.matchedExpectations) {
final log = jsonEncode({
'name': '${options.suiteName}/${test.name}',
'configuration': options.configurationName,
'result': test.matchedExpectations ? 'Pass' : 'Fail',
'log': test.output,
});
logs.add(log);
}
}
// Ensure the directory URI ends with a path separator.
var logDir = Directory.fromUri(options.logDir!).uri;
var resultJsonUri = logDir.resolve('results.json');
var logsJsonUri = logDir.resolve('logs.json');
File.fromUri(resultJsonUri)
.writeAsStringSync(results.map((s) => '$s\n').join(), flush: true);
File.fromUri(logsJsonUri)
.writeAsStringSync(logs.map((s) => '$s\n').join(), flush: true);
print('log files emitted to ${resultJsonUri} and ${logsJsonUri}');
}