blob: 768fa141fd3d6bf628326a84a2a4c19e26bfd875 [file] [log] [blame]
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.
import 'dart:async' show Timer;
import 'dart:convert' show jsonEncode;
import 'dart:io' show File, Platform, exitCode;
import 'dart:isolate' show Isolate, ReceivePort, SendPort;
import 'package:args/args.dart' show ArgParser;
import 'package:testing/src/chain.dart' show CreateContext, Result, Step;
import 'package:testing/src/expectation.dart' show Expectation;
import 'package:testing/src/log.dart' show Logger;
import 'package:testing/src/run.dart' show runMe;
import 'package:testing/src/suite.dart' as testing show Suite;
import 'package:testing/src/test_description.dart' show TestDescription;
import 'fasta/expression_suite.dart' as expression show createContext;
import 'fasta/fast_strong_suite.dart' as fast_strong show createContext;
import 'fasta/incremental_suite.dart' as incremental show createContext;
import 'fasta/messages_suite.dart' as messages show createContext;
import 'fasta/outline_suite.dart' as outline show createContext;
import 'fasta/strong_tester.dart' as strong show createContext;
import 'fasta/text_serialization_tester.dart' as text_serialization
show createContext;
import 'fasta/textual_outline_suite.dart' as textual_outline show createContext;
import 'fasta/weak_suite.dart' as weak show createContext;
import 'incremental_bulk_compiler_smoke_suite.dart' as incremental_bulk_compiler
show createContext;
import 'incremental_load_from_dill_suite.dart' as incremental_load
show createContext;
import 'lint_suite.dart' as lint show createContext;
import 'old_dill_suite.dart' as old_dill show createContext;
import 'parser_suite.dart' as parser show createContext;
import 'spelling_test_not_src_suite.dart' as spelling_not_src
show createContext;
import 'spelling_test_src_suite.dart' as spelling_src show createContext;
const suiteNamePrefix = "pkg/front_end/test";
class Options {
final String configurationName;
final bool verbose;
final bool printFailureLog;
final Uri outputDirectory;
final String testFilter;
Options(this.configurationName, this.verbose, this.printFailureLog,
this.outputDirectory, this.testFilter);
static Options parse(List<String> args) {
var parser = new ArgParser()
..addOption("named-configuration",
abbr: "n",
help: "configuration name to use for emitting json result files")
..addOption("output-directory",
help: "directory to which results.json and logs.json are written")
..addFlag("verbose",
abbr: "v", help: "print additional information", defaultsTo: false)
..addFlag("print",
abbr: "p", help: "print failure logs", defaultsTo: false);
var parsedArguments = parser.parse(args);
String outputPath = parsedArguments["output-directory"] ?? ".";
Uri outputDirectory = Uri.base.resolveUri(Uri.directory(outputPath));
String filter;
if (parsedArguments.rest.length == 1) {
filter = parsedArguments.rest.single;
if (filter.startsWith("$suiteNamePrefix/")) {
filter = filter.substring(suiteNamePrefix.length + 1);
}
}
return Options(
parsedArguments["named-configuration"],
parsedArguments["verbose"],
parsedArguments["print"],
outputDirectory,
filter);
}
}
class ResultLogger implements Logger {
final String prefix;
final bool verbose;
final bool printFailureLog;
final SendPort resultsPort;
final SendPort logsPort;
final Map<String, Stopwatch> stopwatches = {};
final String configurationName;
final Set<String> seenTests = {};
ResultLogger(this.prefix, this.resultsPort, this.logsPort, this.verbose,
this.printFailureLog, this.configurationName);
String getTestName(TestDescription description) {
return "$prefix/${description.shortName}";
}
@override
void logMessage(Object message) {}
@override
void logNumberedLines(String text) {}
@override
void logProgress(String message) {}
@override
void logStepComplete(int completed, int failed, int total,
testing.Suite suite, TestDescription description, Step step) {}
@override
void logStepStart(int completed, int failed, int total, testing.Suite suite,
TestDescription description, Step step) {}
@override
void logSuiteStarted(testing.Suite suite) {}
@override
void logSuiteComplete(testing.Suite suite) {}
handleTestResult(TestDescription testDescription, Result result,
String fullSuiteName, bool matchedExpectations) {
String testName = getTestName(testDescription);
String suite = "pkg";
String shortTestName = testName.substring(suite.length + 1);
resultsPort.send(jsonEncode({
"name": testName,
"configuration": configurationName,
"suite": suite,
"test_name": shortTestName,
"time_ms": stopwatches[testName].elapsedMilliseconds,
"expected": "Pass",
"result": matchedExpectations ? "Pass" : "Fail",
"matches": matchedExpectations,
}));
if (!matchedExpectations) {
String failureLog = result.log;
if (result.error != null) {
failureLog = "$failureLog\n\n${result.error}";
}
if (result.trace != null) {
failureLog = "$failureLog\n\n${result.trace}";
}
failureLog = "$failureLog\n\nRe-run this test: dart "
"pkg/front_end/test/unit_test_suites.dart -p $testName";
String outcome = "${result.outcome}";
logsPort.send(jsonEncode({
"name": testName,
"configuration": configurationName,
"result": outcome,
"log": failureLog,
}));
if (printFailureLog) {
print('FAILED: $testName: $outcome');
print(failureLog);
}
}
if (verbose) {
String result = matchedExpectations ? "PASS" : "FAIL";
print("${testName}: ${result}");
}
}
void logTestStart(int completed, int failed, int total, testing.Suite suite,
TestDescription description) {
String name = getTestName(description);
stopwatches[name] = Stopwatch()..start();
}
@override
void logTestComplete(int completed, int failed, int total,
testing.Suite suite, TestDescription description) {}
@override
void logUncaughtError(error, StackTrace stackTrace) {}
void logExpectedResult(testing.Suite suite, TestDescription description,
Result result, Set<Expectation> expectedOutcomes) {
handleTestResult(description, result, prefix, true);
}
@override
void logUnexpectedResult(testing.Suite suite, TestDescription description,
Result result, Set<Expectation> expectedOutcomes) {
// The test framework (pkg/testing) calls the logger with an unexpected
// results a second time to create a summary. We ignore the second call
// here.
String testName = getTestName(description);
if (seenTests.contains(testName)) return;
seenTests.add(testName);
handleTestResult(description, result, prefix, false);
}
}
class Suite {
final String name;
final CreateContext createContext;
final String testingRootPath;
final String path;
final int shardCount;
final int shard;
final String prefix;
const Suite(this.name, this.createContext, this.testingRootPath,
{this.path, this.shardCount: 1, this.shard: 0, String prefix})
: prefix = prefix ?? name;
}
const List<Suite> suites = [
const Suite(
"fasta/expression", expression.createContext, "../../testing.json"),
const Suite("fasta/outline", outline.createContext, "../../testing.json"),
const Suite(
"fasta/fast_strong", fast_strong.createContext, "../../testing.json"),
const Suite(
"fasta/incremental", incremental.createContext, "../../testing.json"),
const Suite("fasta/messages", messages.createContext, "../../testing.json"),
const Suite("fasta/text_serialization1", text_serialization.createContext,
"../../testing.json",
path: "fasta/text_serialization_tester.dart",
shardCount: 4,
shard: 0,
prefix: "fasta/text_serialization"),
const Suite("fasta/text_serialization2", text_serialization.createContext,
"../../testing.json",
path: "fasta/text_serialization_tester.dart",
shardCount: 4,
shard: 1,
prefix: "fasta/text_serialization"),
const Suite("fasta/text_serialization3", text_serialization.createContext,
"../../testing.json",
path: "fasta/text_serialization_tester.dart",
shardCount: 4,
shard: 2,
prefix: "fasta/text_serialization"),
const Suite("fasta/text_serialization4", text_serialization.createContext,
"../../testing.json",
path: "fasta/text_serialization_tester.dart",
shardCount: 4,
shard: 3,
prefix: "fasta/text_serialization"),
const Suite("fasta/strong1", strong.createContext, "../../testing.json",
path: "fasta/strong_tester.dart",
shardCount: 4,
shard: 0,
prefix: "fasta/strong"),
const Suite("fasta/strong2", strong.createContext, "../../testing.json",
path: "fasta/strong_tester.dart",
shardCount: 4,
shard: 1,
prefix: "fasta/strong"),
const Suite("fasta/strong3", strong.createContext, "../../testing.json",
path: "fasta/strong_tester.dart",
shardCount: 4,
shard: 2,
prefix: "fasta/strong"),
const Suite("fasta/strong4", strong.createContext, "../../testing.json",
path: "fasta/strong_tester.dart",
shardCount: 4,
shard: 3,
prefix: "fasta/strong"),
const Suite("incremental_bulk_compiler_smoke",
incremental_bulk_compiler.createContext, "../testing.json"),
const Suite("incremental_load_from_dill", incremental_load.createContext,
"../testing.json"),
const Suite("lint", lint.createContext, "../testing.json"),
const Suite("old_dill", old_dill.createContext, "../testing.json"),
const Suite("parser", parser.createContext, "../testing.json"),
const Suite("spelling_test_not_src", spelling_not_src.createContext,
"../testing.json"),
const Suite(
"spelling_test_src", spelling_src.createContext, "../testing.json"),
const Suite("fasta/weak", weak.createContext, "../../testing.json"),
const Suite("fasta/textual_outline", textual_outline.createContext,
"../../testing.json"),
];
const Duration timeoutDuration = Duration(minutes: 25);
class SuiteConfiguration {
final String name;
final SendPort resultsPort;
final SendPort logsPort;
final bool verbose;
final bool printFailureLog;
final String configurationName;
final String testFilter;
const SuiteConfiguration(
this.name,
this.resultsPort,
this.logsPort,
this.verbose,
this.printFailureLog,
this.configurationName,
this.testFilter);
}
void runSuite(SuiteConfiguration configuration) {
Suite suite = suites.where((s) => s.name == configuration.name).single;
String name = suite.prefix;
String fullSuiteName = "$suiteNamePrefix/$name";
Uri suiteUri = Platform.script.resolve(suite.path ?? "${name}_suite.dart");
ResultLogger logger = ResultLogger(
fullSuiteName,
configuration.resultsPort,
configuration.logsPort,
configuration.verbose,
configuration.printFailureLog,
configuration.configurationName);
runMe(
<String>[if (configuration.testFilter != null) configuration.testFilter],
suite.createContext,
me: suiteUri,
configurationPath: suite.testingRootPath,
logger: logger,
shards: suite.shardCount,
shard: suite.shard);
}
void writeLinesToFile(Uri uri, List<String> lines) async {
await File.fromUri(uri).writeAsString(lines.map((line) => "$line\n").join());
}
main([List<String> arguments = const <String>[]]) async {
List<String> results = [];
List<String> logs = [];
Options options = Options.parse(arguments);
ReceivePort resultsPort = new ReceivePort()
..listen((resultEntry) => results.add(resultEntry));
ReceivePort logsPort = new ReceivePort()
..listen((logEntry) => logs.add(logEntry));
List<Future<bool>> futures = [];
// Run test suites and record the results and possible failure logs.
for (Suite suite in suites) {
String name = suite.name;
String filter = options.testFilter;
if (filter != null) {
// Skip suites that are not hit by the test filter, is there is one.
if (!filter.startsWith(suite.prefix)) {
continue;
}
// Remove the 'fasta/' from filters, if there, because it is not used
// in the name defined in testing.json.
if (filter.startsWith("fasta/")) {
filter = filter.substring("fasta/".length);
}
}
// Start the test suite in a new isolate.
ReceivePort exitPort = new ReceivePort();
SuiteConfiguration configuration = SuiteConfiguration(
name,
resultsPort.sendPort,
logsPort.sendPort,
options.verbose,
options.printFailureLog,
options.configurationName,
filter);
Future future = Future<bool>(() async {
Stopwatch stopwatch = Stopwatch()..start();
print("Running suite $name");
Isolate isolate = await Isolate.spawn<SuiteConfiguration>(
runSuite, configuration,
onExit: exitPort.sendPort);
bool timedOut = false;
Timer timer = Timer(timeoutDuration, () {
timedOut = true;
print("Suite $name timed out after "
"${timeoutDuration.inMilliseconds}ms");
isolate.kill(priority: Isolate.immediate);
});
await exitPort.first;
timer.cancel();
if (!timedOut) {
int seconds = stopwatch.elapsedMilliseconds ~/ 1000;
print("Suite $name finished (took ${seconds} seconds)");
}
return timedOut;
});
futures.add(future);
}
// Wait for isolates to terminate and clean up.
Iterable<bool> timeouts = await Future.wait(futures);
resultsPort.close();
logsPort.close();
// Write results.json and logs.json.
Uri resultJsonUri = options.outputDirectory.resolve("results.json");
Uri logsJsonUri = options.outputDirectory.resolve("logs.json");
await writeLinesToFile(resultJsonUri, results);
await writeLinesToFile(logsJsonUri, logs);
print("Log files written to ${resultJsonUri.toFilePath()} and"
" ${logsJsonUri.toFilePath()}");
// Return with exit code 1 if at least one suite timed out.
bool timeout = timeouts.any((timeout) => timeout);
if (timeout) {
exitCode = 1;
} else {
// The testing framework (package:testing) sets the exitCode to `1` if any
// test failed, so we reset it here to indicate that the test runner was
// successful.
exitCode = 0;
}
}