blob: 8ab797bcacbdab214f65c317f7d828bc38885a02 [file] [log] [blame]
// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the file.
library testing.chain;
import 'dart:async' show Future, Stream;
import 'dart:convert' show json, JsonEncoder;
import 'dart:io' show Directory, File, FileSystemEntity, exitCode;
import 'suite.dart' show Suite;
import '../testing.dart' show FileBasedTestDescription, TestDescription;
import 'test_dart/status_file_parser.dart'
show ReadTestExpectations, TestExpectations;
import 'zone_helper.dart' show runGuarded;
import 'error_handling.dart' show withErrorHandling;
import 'log.dart' show Logger, StdoutLogger, splitLines;
import 'multitest.dart' show MultitestTransformer, isError;
import 'expectation.dart' show Expectation, ExpectationSet;
typedef Future<ChainContext> CreateContext(
Chain suite, Map<String, String> environment);
/// A test suite for tool chains, for example, a compiler.
class Chain extends Suite {
final Uri source;
final Uri uri;
final List<RegExp> pattern;
final List<RegExp> exclude;
final bool processMultitests;
Chain(String name, String kind, this.source, this.uri, Uri statusFile,
this.pattern, this.exclude, this.processMultitests)
: super(name, kind, statusFile);
factory Chain.fromJsonMap(Uri base, Map json, String name, String kind) {
Uri source = base.resolve(json["source"]);
String path = json["path"];
if (!path.endsWith("/")) {
path += "/";
Uri uri = base.resolve(path);
Uri statusFile = base.resolve(json["status"]);
List<RegExp> pattern =
json["pattern"].map<RegExp>((p) => new RegExp(p)).toList();
List<RegExp> exclude =
json["exclude"].map<RegExp>((p) => new RegExp(p)).toList();
bool processMultitests = json["process-multitests"] ?? false;
return new Chain(name, kind, source, uri, statusFile, pattern, exclude,
void writeImportOn(StringSink sink) {
sink.write("import '");
sink.write("' as ");
void writeClosureOn(StringSink sink) {
sink.write("await runChain(");
sink.writeln(".createContext, environment, selectors, r'''");
const String jsonExtraIndent = " ";
sink.writeAll(splitLines(new JsonEncoder.withIndent(" ").convert(this)),
Map toJson() {
return {
"name": name,
"kind": kind,
"source": "$source",
"path": "$uri",
"status": "$statusFile",
"process-multitests": processMultitests,
"pattern": []..addAll( r) => r.pattern)),
"exclude": []..addAll( r) => r.pattern)),
abstract class ChainContext {
const ChainContext();
List<Step> get steps;
ExpectationSet get expectationSet => ExpectationSet.Default;
Future<Null> run(Chain suite, Set<String> selectors,
{int shards = 1,
int shard = 0,
Logger logger: const StdoutLogger()}) async {
assert(shards >= 1, "Invalid shards count: $shards");
assert(0 <= shard && shard < shards,
"Invalid shard index: $shard, not in range [0,$shards[.");
List<String> partialSelectors = selectors
.where((s) => s.endsWith('...'))
.map((s) => s.substring(0, s.length - 3))
TestExpectations expectations = await ReadTestExpectations(
<String>[suite.statusFile!.toFilePath()], {}, expectationSet);
Stream<TestDescription> stream = list(suite);
if (suite.processMultitests) {
stream = stream.transform(new MultitestTransformer());
List<TestDescription> descriptions = await stream.toList();
if (shards > 1) {
List<TestDescription> shardDescriptions = [];
for (int index = 0; index < descriptions.length; index++) {
if (index % shards == shard) {
descriptions = shardDescriptions;
Map<TestDescription, Result> unexpectedResults =
<TestDescription, Result>{};
Map<TestDescription, Set<Expectation>> unexpectedOutcomes =
<TestDescription, Set<Expectation>>{};
int completed = 0;
List<Future> futures = <Future>[];
for (TestDescription description in descriptions) {
String selector = "${}/${description.shortName}";
if (selectors.isNotEmpty &&
!selectors.contains(selector) &&
!selectors.contains( &&
!partialSelectors.any((s) => selector.startsWith(s))) {
final Set<Expectation> expectedOutcomes = processExpectedOutcomes(
expectations.expectations(description.shortName), description);
final StringBuffer sb = new StringBuffer();
final Step? lastStep = steps.isNotEmpty ? steps.last : null;
final Iterator<Step> iterator = steps.iterator;
Result? result;
// Records the outcome of the last step that was run.
Step? lastStepRun;
/// Performs one step of [iterator].
/// If `step.isAsync` is true, the corresponding step is said to be
/// asynchronous.
/// If a step is asynchronous the future returned from this function will
/// complete after the first asynchronous step is scheduled. This
/// allows us to start processing the next test while an external process
/// completes as steps can be interleaved. To ensure all steps are
/// completed, wait for [futures].
/// Otherwise, the future returned will complete when all steps are
/// completed. This ensures that tests are run in sequence without
/// interleaving steps.
Future doStep(dynamic input) async {
Future future;
bool isAsync = false;
if (iterator.moveNext()) {
Step step = iterator.current;
lastStepRun = step;
isAsync = step.isAsync;
logger.logStepStart(completed, unexpectedResults.length,
descriptions.length, suite, description, step);
// TODO(ahe): It's important to share the zone error reporting zone
// between all the tasks. Otherwise, if a future completes with an
// error in one zone, and gets stored, it becomes an uncaught error
// in other zones (this happened in createPlatform).
future = runGuarded(() async {
try {
return await, this);
} catch (error, trace) {
return step.unhandledError(error, trace);
}, printLineOnStdout: sb.writeln);
} else {
future = new Future.value(null);
future = future.then((_currentResult) async {
Result? currentResult = _currentResult;
if (currentResult != null) {
logger.logStepComplete(completed, unexpectedResults.length,
descriptions.length, suite, description, lastStepRun!);
result = currentResult;
if (currentResult.outcome == Expectation.Pass) {
// The input to the next step is the output of this step.
return doStep(result!.output);
await cleanUp(description, result!);
result =
processTestResult(description, result!, lastStep == lastStepRun);
if (!expectedOutcomes.contains(result!.outcome) &&
!expectedOutcomes.contains(result!.outcome.canonical)) {
unexpectedResults[description] = result!;
unexpectedOutcomes[description] = expectedOutcomes;
suite, description, result!, expectedOutcomes);
exitCode = 1;
} else {
suite, description, result!, expectedOutcomes);
logger.logTestComplete(++completed, unexpectedResults.length,
descriptions.length, suite, description);
if (isAsync) {
return null;
} else {
return future;
logger.logTestStart(completed, unexpectedResults.length,
descriptions.length, suite, description);
// The input of the first step is [description].
await doStep(description);
await Future.wait(futures);
if (unexpectedResults.isNotEmpty) {
unexpectedResults.forEach((TestDescription description, Result result) {
suite, description, result, unexpectedOutcomes[description]!);
print("${unexpectedResults.length} failed:");
unexpectedResults.forEach((TestDescription description, Result result) {
print("${}/${description.shortName}: ${result.outcome}");
Stream<TestDescription> list(Chain suite) async* {
Directory testRoot = new Directory.fromUri(suite.uri);
if (await testRoot.exists()) {
Stream<FileSystemEntity> files =
testRoot.list(recursive: true, followLinks: false);
await for (FileSystemEntity entity in files) {
if (entity is! File) continue;
String path = entity.uri.path;
if (suite.exclude.any((RegExp r) => path.contains(r))) continue;
if (suite.pattern.any((RegExp r) => path.contains(r))) {
yield new FileBasedTestDescription(suite.uri, entity);
} else {
throw "${suite.uri} isn't a directory";
Set<Expectation> processExpectedOutcomes(
Set<Expectation> outcomes, TestDescription description) {
return outcomes;
Result processTestResult(
TestDescription description, Result result, bool last) {
if (description is FileBasedTestDescription &&
description.multitestExpectations != null) {
if (isError(description.multitestExpectations!)) {
result =
toNegativeTestResult(result, description.multitestExpectations);
} else if (last && description.shortName.endsWith("negative_test")) {
if (result.outcome == Expectation.Pass) {
result.addLog("Negative test didn't report an error.\n");
} else if (result.outcome == Expectation.Fail) {
result.addLog("Negative test reported an error as expected.\n");
result = toNegativeTestResult(result);
return result;
Result toNegativeTestResult(Result result, [Set<String>? expectations]) {
Expectation outcome = result.outcome;
if (outcome == Expectation.Pass) {
if (expectations == null) {
outcome = Expectation.Fail;
} else if (expectations.contains("compile-time error")) {
outcome = expectationSet["MissingCompileTimeError"];
} else if (expectations.contains("runtime error") ||
expectations.contains("dynamic type error")) {
outcome = expectationSet["MissingRuntimeError"];
} else {
outcome = Expectation.Fail;
} else if (outcome == Expectation.Fail) {
outcome = Expectation.Pass;
return result.copyWithOutcome(outcome);
Future<void> cleanUp(TestDescription description, Result result) async {}
Future<void> postRun() async {}
abstract class Step<I, O, C extends ChainContext> {
const Step();
String get name;
/// Sets this (*and effectively subsequent*) test step(s) as async.
/// TL;DR: Either set to false, or only set to true when this and all
/// subsequent steps can run intertwined with another test.
/// Details:
/// A single test (TestDescription) can have several steps (Step).
/// When running a test the first step is executed, and when that step is done
/// the next step is executed by the now-ending step.
/// When isAsync is false each step returns a future which is awaited,
/// effectivly meaning that only a single test is run at a time.
/// When isAsync is true that step doesn't return a future (but adds it's
/// future to a list which is awaited before sending an 'entire suite done'
/// message), meaning that the next test can start before the step is
/// finished. As the next step in the test only starts after the current
/// step finishes, that also means that the next test can start - and run
/// intertwined with - a subsequent step even if such a subsequent step has
/// isAsync set to false.
bool get isAsync => false;
Future<Result<O>> run(I input, C context);
Result<O> unhandledError(error, StackTrace trace) {
return new Result<O>.crash(error, trace);
Result<O> pass(O output) => new Result<O>.pass(output);
Result<O> crash(error, StackTrace trace) => new Result<O>.crash(error, trace);
Result<O> fail(O output, [error, StackTrace? trace]) {
return new Result<O>.fail(output, error, trace);
class Result<O> {
final O? output;
final Expectation outcome;
final error;
final StackTrace? trace;
final List<String> logs = <String>[];
/// If set, running the test with '-D$autoFixCommand' will automatically
/// update the test to match new expectations.
final String? autoFixCommand;
/// If set, the test can be fixed by running
/// dart pkg/front_end/tool/update_expectations.dart
final bool canBeFixWithUpdateExpectations;
Result(this.output, this.outcome, this.error,
this.canBeFixWithUpdateExpectations: false});
Result.pass(O output) : this(output, Expectation.Pass, null);
Result.crash(error, StackTrace trace)
: this(null, Expectation.Crash, error, trace: trace); output, [error, StackTrace? trace])
: this(output, Expectation.Fail, error, trace: trace);
bool get isPass => outcome == Expectation.Pass;
String get log => logs.join();
void addLog(String log) {
Result<O> copyWithOutcome(Expectation outcome) {
return new Result<O>(output, outcome, error, trace: trace)
Result<O2> copyWithOutput<O2>(O2 output) {
return new Result<O2>(output, outcome, error,
trace: trace,
autoFixCommand: autoFixCommand,
canBeFixWithUpdateExpectations: canBeFixWithUpdateExpectations)
/// This is called from generated code.
Future<Null> runChain(CreateContext f, Map<String, String> environment,
Set<String> selectors, String jsonText) {
return withErrorHandling(() async {
Chain suite =
new Suite.fromJsonMap(Uri.base, json.decode(jsonText)) as Chain;
print("Running ${}");
ChainContext context = await f(suite, environment);
return, selectors);