blob: f8eba4b09e79c096ee9e99dbdef9c4676fa8f9ad [file] [log] [blame]
// Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.
import 'dart:async' show Timer;
import 'dart:convert' show jsonEncode;
import 'dart:io' show File, Platform, exitCode;
import 'dart:isolate' show Isolate, ReceivePort, SendPort;
import 'package:args/args.dart' show ArgParser;
import 'package:testing/src/chain.dart' show CreateContext, Result, Step;
import 'package:testing/src/expectation.dart' show Expectation;
import 'package:testing/src/log.dart' show Logger;
import 'package:testing/src/run.dart' show runMe;
import 'package:testing/src/suite.dart' as testing show Suite;
import 'package:testing/src/test_description.dart' show TestDescription;
import 'dartdoctest_suite.dart' as dartdoctest show createContext;
import 'fasta/expression_suite.dart' as expression show createContext;
import 'fasta/incremental_dartino_suite.dart' as incremental_dartino
show createContext;
import 'fasta/messages_suite.dart' as messages show createContext;
import 'fasta/modular_suite.dart' as modular show createContext;
import 'fasta/outline_suite.dart' as outline show createContext;
import 'fasta/strong_suite.dart' as strong show createContext;
import 'fasta/textual_outline_suite.dart' as textual_outline show createContext;
import 'fasta/weak_suite.dart' as weak show createContext;
import 'incremental_bulk_compiler_smoke_suite.dart' as incremental_bulk_compiler
show createContext;
import 'incremental_suite.dart' as incremental show createContext;
import 'lint_suite.dart' as lint show createContext;
import 'outline_extractor_suite.dart' as outline_extractor show createContext;
import 'parser_all_suite.dart' as parserAll show createContext;
import 'parser_equivalence_suite.dart' as parserEquivalence show createContext;
import 'parser_suite.dart' as parser show createContext;
import 'spelling_test_not_src_suite.dart' as spelling_not_src
show createContext;
import 'spelling_test_src_suite.dart' as spelling_src show createContext;
const suiteNamePrefix = "pkg/front_end/test";
int getDefaultThreads() {
int numberOfWorkers = 1;
if (Platform.numberOfProcessors > 2) {
numberOfWorkers = Platform.numberOfProcessors - 1;
}
return numberOfWorkers;
}
class Options {
final String? configurationName;
final bool verbose;
final bool printFailureLog;
final Uri outputDirectory;
final String? testFilter;
final List<String> environmentOptions;
final int shardCount;
final int shard;
final bool skipTestsThatRequireGit;
final bool onlyTestsThatRequireGit;
final int numberOfWorkers;
Options(
this.configurationName,
this.verbose,
this.printFailureLog,
this.outputDirectory,
this.testFilter,
this.environmentOptions, {
required this.shardCount,
required this.shard,
required this.skipTestsThatRequireGit,
required this.onlyTestsThatRequireGit,
required this.numberOfWorkers,
});
static Options parse(List<String> args) {
var parser = new ArgParser()
..addOption("named-configuration",
abbr: "n",
help: "configuration name to use for emitting json result files")
..addOption("output-directory",
help: "directory to which results.json and logs.json are written")
..addFlag("verbose",
abbr: "v", help: "print additional information", defaultsTo: false)
..addFlag("print",
abbr: "p", help: "print failure logs", defaultsTo: false)
..addMultiOption('environment',
abbr: 'D', help: "environment options for the test suite")
..addOption("tasks",
abbr: "j",
help: "The number of parallel tasks to run.",
defaultsTo: "${getDefaultThreads()}")
..addOption("shards", help: "Number of shards", defaultsTo: "1")
..addOption("shard", help: "Which shard to run", defaultsTo: "1")
..addFlag("skipTestsThatRequireGit",
help: "Whether to skip tests that require git to run",
defaultsTo: false)
..addFlag("onlyTestsThatRequireGit",
help: "Whether to only run tests that require git",
defaultsTo: false);
var parsedOptions = parser.parse(args);
String outputPath = parsedOptions["output-directory"] ?? ".";
Uri outputDirectory = Uri.base.resolveUri(Uri.directory(outputPath));
bool verbose = parsedOptions["verbose"];
String? filter;
if (parsedOptions.rest.length == 1) {
filter = parsedOptions.rest.single;
if (filter.startsWith("$suiteNamePrefix/")) {
filter = filter.substring(suiteNamePrefix.length + 1);
}
}
String tasksString = parsedOptions["tasks"];
int? tasks = int.tryParse(tasksString);
if (tasks == null || tasks < 1) {
throw "--tasks (-j) has to be an integer >= 1";
}
String shardsString = parsedOptions["shards"];
int? shardCount = int.tryParse(shardsString);
if (shardCount == null || shardCount < 1) {
throw "--shards has to be an integer >= 1";
}
String shardString = parsedOptions["shard"];
int? shard = int.tryParse(shardString);
if (shard == null || shard < 1 || shard > shardCount) {
throw "--shard has to be an integer >= 1 (and <= --shards)";
}
bool skipTestsThatRequireGit = parsedOptions["skipTestsThatRequireGit"];
bool onlyTestsThatRequireGit = parsedOptions["onlyTestsThatRequireGit"];
if (skipTestsThatRequireGit && onlyTestsThatRequireGit) {
throw "Only one of --skipTestsThatRequireGit and "
"--onlyTestsThatRequireGit can be provided.";
}
if (verbose) {
print("NOTE: Created with options\n "
"${parsedOptions["named-configuration"]},\n "
"${verbose},\n "
"${parsedOptions["print"]},\n "
"${outputDirectory},\n "
"${filter},\n "
"${parsedOptions['environment']},\n "
"shardCount: ${shardCount},\n "
"shard: ${shard - 1 /* make it 0-indexed */},\n "
"onlyTestsThatRequireGit: ${onlyTestsThatRequireGit},\n "
"skipTestsThatRequireGit: ${skipTestsThatRequireGit},\n "
"numberOfWorkers: ${tasks}");
}
return Options(
parsedOptions["named-configuration"],
verbose,
parsedOptions["print"],
outputDirectory,
filter,
parsedOptions['environment'],
shardCount: shardCount,
shard: shard - 1 /* make it 0-indexed */,
onlyTestsThatRequireGit: onlyTestsThatRequireGit,
skipTestsThatRequireGit: skipTestsThatRequireGit,
numberOfWorkers: tasks,
);
}
}
class ResultLogger implements Logger {
final String prefix;
final bool verbose;
final bool printFailureLog;
final SendPort resultsPort;
final SendPort logsPort;
final Map<String, Stopwatch> stopwatches = {};
final String? configurationName;
final Set<String> seenTests = {};
bool gotFrameworkError = false;
ResultLogger(this.prefix, this.resultsPort, this.logsPort, this.verbose,
this.printFailureLog, this.configurationName);
String getTestName(TestDescription description) {
return "$prefix/${description.shortName}";
}
@override
void logMessage(Object message) {}
@override
void logNumberedLines(String text) {}
@override
void logProgress(String message) {}
@override
void logStepComplete(int completed, int failed, int total,
testing.Suite suite, TestDescription description, Step step) {}
@override
void logStepStart(int completed, int failed, int total, testing.Suite suite,
TestDescription description, Step step) {}
@override
void logSuiteStarted(testing.Suite suite) {}
@override
void logSuiteComplete(testing.Suite suite) {}
void handleTestResult(
testing.Suite suite,
TestDescription testDescription,
Result result,
String fullSuiteName,
bool matchedExpectations,
Set<Expectation> expectedOutcomes) {
String testName = getTestName(testDescription);
String suiteName = "pkg";
String shortTestName = testName.substring(suiteName.length + 1);
resultsPort.send(jsonEncode({
"name": testName,
"configuration": configurationName,
"suite": suiteName,
"test_name": shortTestName,
"time_ms": stopwatches[testName]!.elapsedMilliseconds,
"expected": "Pass",
"result": matchedExpectations ? "Pass" : "Fail",
"matches": matchedExpectations,
}));
if (!matchedExpectations) {
StringBuffer sb = new StringBuffer();
if (printFailureLog) {
String outcome = "${result.outcome}";
sb.write("FAILED: $testName: $outcome");
}
sb.write(result.log);
if (result.error != null) {
sb.write("\n\n${result.error}");
}
if (result.trace != null) {
sb.write("\n\n${result.trace}");
}
sb.write("\n\nTo re-run this test, run:");
sb.write("\n\n dart pkg/front_end/test/unit_test_suites.dart -p "
"$testName");
if (result.autoFixCommand != null) {
sb.write("\n\nTo automatically update the test expectations, run:");
sb.write("\n\n dart pkg/front_end/test/unit_test_suites.dart -p "
"$testName -D${result.autoFixCommand}");
if (result.canBeFixWithUpdateExpectations) {
sb.write('\n\nTo update test expectations for all tests at once, '
'run:');
sb.write('\n\n dart pkg/front_end/tool/update_expectations.dart');
sb.write('\n\nNote that this takes a long time and should only be '
'used when many tests need updating.\n');
}
}
if (result.outcome == Expectation.Pass) {
String expectedString =
expectedOutcomes.map((e) => e.toString()).join(", ");
sb.write("\n\nThe test passed, but wasn't expected to. "
"You should update the status file for this test."
"\nThere's a status entry looking something like"
"\n\n ${testDescription.shortName}: ${expectedString}"
"\n\nwhich should be removed."
"\n\nThe status file is ${suite.statusFile}.");
} else if (result.autoFixCommand == null) {
String expectedString =
expectedOutcomes.map((e) => e.toString()).join(", ");
sb.write("\n\nThe test has outcome ${result.outcome}, "
"but was expected to have outcome(s) ${expectedOutcomes}. "
"You might have to update the status file to the new outcome"
"\nThere's a status entry looking something like"
"\n\n ${testDescription.shortName}: ${expectedString}"
"\n\nwhich should be updated."
"\n\nThe status file is ${suite.statusFile}.");
}
String failureLog = sb.toString();
String outcome = "${result.outcome}";
logsPort.send(jsonEncode({
"name": testName,
"configuration": configurationName,
"result": outcome,
"log": failureLog,
}));
if (printFailureLog) {
print(failureLog);
}
}
if (verbose) {
String result = matchedExpectations ? "PASS" : "FAIL";
print("${testName}: ${result}");
}
}
@override
void logTestStart(int completed, int failed, int total, testing.Suite suite,
TestDescription description) {
String name = getTestName(description);
stopwatches[name] = Stopwatch()..start();
}
@override
void logTestComplete(int completed, int failed, int total,
testing.Suite suite, TestDescription description) {}
@override
void logUncaughtError(error, StackTrace stackTrace) {}
@override
void logExpectedResult(testing.Suite suite, TestDescription description,
Result result, Set<Expectation> expectedOutcomes) {
handleTestResult(
suite, description, result, prefix, true, expectedOutcomes);
}
@override
void logUnexpectedResult(testing.Suite suite, TestDescription description,
Result result, Set<Expectation> expectedOutcomes) {
// The test framework (pkg/testing) calls the logger with an unexpected
// results a second time to create a summary. We ignore the second call
// here.
String testName = getTestName(description);
if (seenTests.contains(testName)) return;
seenTests.add(testName);
handleTestResult(
suite, description, result, prefix, false, expectedOutcomes);
}
@override
void noticeFrameworkCatchError(error, StackTrace stackTrace) {
gotFrameworkError = true;
}
}
class Suite {
final CreateContext createContext;
final String testingRootPath;
final String? path;
final int shardCount;
final String prefix;
final bool requiresGit;
const Suite(
this.prefix,
this.createContext,
this.testingRootPath, {
required this.shardCount,
this.path,
this.requiresGit: false,
});
}
const List<Suite> suites = [
const Suite(
"dartdoctest",
dartdoctest.createContext,
"../testing.json",
shardCount: 1,
),
const Suite(
"fasta/expression",
expression.createContext,
"../../testing.json",
shardCount: 1,
),
const Suite(
"fasta/outline",
outline.createContext,
"../../testing.json",
shardCount: 2,
),
const Suite(
"fasta/incremental_dartino",
incremental_dartino.createContext,
"../../testing.json",
shardCount: 1,
),
const Suite(
"fasta/messages",
messages.createContext,
"../../testing.json",
shardCount: 1,
requiresGit: true,
),
const Suite(
"fasta/strong",
strong.createContext,
"../../testing.json",
path: "fasta/strong_suite.dart",
shardCount: 2,
),
const Suite(
"incremental_bulk_compiler_smoke",
incremental_bulk_compiler.createContext,
"../testing.json",
shardCount: 1,
),
const Suite(
"incremental",
incremental.createContext,
"../testing.json",
shardCount: 2,
),
const Suite(
"lint",
lint.createContext,
"../testing.json",
shardCount: 1,
requiresGit: true,
),
const Suite(
"parser",
parser.createContext,
"../testing.json",
shardCount: 1,
),
const Suite(
"parser_equivalence",
parserEquivalence.createContext,
"../testing.json",
shardCount: 1,
),
const Suite(
"parser_all",
parserAll.createContext,
"../testing.json",
shardCount: 4,
requiresGit:
true /* technically not true, but tests *many* more files
than in test_matrix.json file set */
,
),
const Suite(
"spelling_test_not_src",
spelling_not_src.createContext,
"../testing.json",
shardCount: 1,
requiresGit: true,
),
const Suite(
"spelling_test_src",
spelling_src.createContext,
"../testing.json",
shardCount: 1,
requiresGit: true,
),
const Suite(
"fasta/modular",
modular.createContext,
"../../testing.json",
path: "fasta/modular_suite.dart",
shardCount: 4,
),
const Suite(
"fasta/weak",
weak.createContext,
"../../testing.json",
path: "fasta/weak_suite.dart",
shardCount: 10,
),
const Suite(
"fasta/textual_outline",
textual_outline.createContext,
"../../testing.json",
shardCount: 1,
),
const Suite(
"outline_extractor",
outline_extractor.createContext,
"../testing.json",
shardCount: 1,
),
];
const Duration timeoutDuration = Duration(minutes: 30);
class SuiteConfiguration {
final Suite suite;
final SendPort resultsPort;
final SendPort logsPort;
final bool verbose;
final bool printFailureLog;
final String? configurationName;
final String? testFilter;
final List<String> environmentOptions;
final int shard;
const SuiteConfiguration(
this.suite,
this.resultsPort,
this.logsPort,
this.verbose,
this.printFailureLog,
this.configurationName,
this.testFilter,
this.environmentOptions,
this.shard,
);
}
Future<void> runSuite(SuiteConfiguration configuration) async {
Suite suite = configuration.suite;
String name = suite.prefix;
String fullSuiteName = "$suiteNamePrefix/$name";
Uri suiteUri = Platform.script.resolve(suite.path ?? "${name}_suite.dart");
if (!new File.fromUri(suiteUri).existsSync()) {
throw "File doesn't exist: $suiteUri";
}
ResultLogger logger = ResultLogger(
fullSuiteName,
configuration.resultsPort,
configuration.logsPort,
configuration.verbose,
configuration.printFailureLog,
configuration.configurationName);
await runMe(
<String>[
if (configuration.testFilter != null) configuration.testFilter!,
for (String option in configuration.environmentOptions) '-D${option}',
],
suite.createContext,
me: suiteUri,
configurationPath: suite.testingRootPath,
logger: logger,
shards: suite.shardCount,
shard: configuration.shard,
);
if (logger.gotFrameworkError) {
throw "Got framework error!";
}
}
Future<void> writeLinesToFile(Uri uri, List<String> lines) async {
await File.fromUri(uri).writeAsString(lines.map((line) => "$line\n").join());
}
Future<void> main([List<String> arguments = const <String>[]]) async {
Stopwatch totalRuntime = new Stopwatch()..start();
List<String> results = [];
List<String> logs = [];
Options options = Options.parse(arguments);
ReceivePort resultsPort = new ReceivePort()
..listen((resultEntry) => results.add(resultEntry));
ReceivePort logsPort = new ReceivePort()
..listen((logEntry) => logs.add(logEntry));
List<Future<bool>> futures = [];
if (options.verbose) {
print("NOTE: Willing to run with ${options.numberOfWorkers} 'workers'");
print("");
}
int numberOfFreeWorkers = options.numberOfWorkers;
// Run test suites and record the results and possible failure logs.
int chunkNum = 0;
for (Suite suite in suites) {
if (options.onlyTestsThatRequireGit && !suite.requiresGit) continue;
if (options.skipTestsThatRequireGit && suite.requiresGit) continue;
String prefix = suite.prefix;
String? filter = options.testFilter;
if (filter != null) {
// Skip suites that are not hit by the test filter, is there is one.
if (!filter.startsWith(prefix)) {
continue;
}
// Remove the 'fasta/' from filters, if there, because it is not used
// in the name defined in testing.json.
if (filter.startsWith("fasta/")) {
filter = filter.substring("fasta/".length);
}
}
for (int shard = 0; shard < suite.shardCount; shard++) {
if (chunkNum++ % options.shardCount != options.shard) continue;
while (numberOfFreeWorkers <= 0) {
// This might not be great design, but it'll work fine.
await Future.delayed(const Duration(milliseconds: 50));
}
numberOfFreeWorkers--;
// Start the test suite in a new isolate.
ReceivePort exitPort = new ReceivePort();
ReceivePort errorPort = new ReceivePort();
SuiteConfiguration configuration = new SuiteConfiguration(
suite,
resultsPort.sendPort,
logsPort.sendPort,
options.verbose,
options.printFailureLog,
options.configurationName,
filter,
options.environmentOptions,
shard);
Future<bool> future = new Future<bool>(() async {
try {
Stopwatch stopwatch = new Stopwatch()..start();
String naming = "$prefix";
if (suite.shardCount > 1) {
naming += " (${shard + 1} of ${suite.shardCount})";
}
print("Running suite $naming");
Isolate isolate = await Isolate.spawn<SuiteConfiguration>(
runSuite, configuration,
onExit: exitPort.sendPort, onError: errorPort.sendPort);
bool timedOutOrCrash = false;
Timer timer = new Timer(timeoutDuration, () {
timedOutOrCrash = true;
print("Suite $naming timed out after "
"${timeoutDuration.inMilliseconds}ms");
isolate.kill(priority: Isolate.immediate);
});
await exitPort.first;
errorPort.close();
List<dynamic> allErrors = await errorPort.toList();
bool gotError = allErrors.isNotEmpty;
if (gotError) {
print("Suite $naming encountered ${allErrors.length} error(s).");
print("Errors:");
for (int i = 0; i < allErrors.length; i++) {
print("-----------");
print("Error #$i:");
print(allErrors[i]);
}
print("-----------");
timedOutOrCrash = true;
}
timer.cancel();
int seconds = stopwatch.elapsedMilliseconds ~/ 1000;
if (!timedOutOrCrash) {
print("Suite $naming finished (took ${seconds} seconds)");
} else {
print("Suite $naming finished badly (see above) "
"(took ${seconds} seconds)");
}
return timedOutOrCrash;
} finally {
numberOfFreeWorkers++;
}
});
futures.add(future);
}
}
// Wait for isolates to terminate and clean up.
Iterable<bool> timeoutsOrCrashes = await Future.wait(futures);
resultsPort.close();
logsPort.close();
// Write results.json and logs.json.
Uri resultJsonUri = options.outputDirectory.resolve("results.json");
Uri logsJsonUri = options.outputDirectory.resolve("logs.json");
await writeLinesToFile(resultJsonUri, results);
await writeLinesToFile(logsJsonUri, logs);
print("Log files written to ${resultJsonUri.toFilePath()} and"
" ${logsJsonUri.toFilePath()}");
print("Entire run took ${totalRuntime.elapsed}.");
// Return with exit code 1 if at least one suite timed out.
bool timedOutOrCrashed = timeoutsOrCrashes.any((timeout) => timeout);
if (timedOutOrCrashed) {
throw "Crashed or timed out. Check stdout for more details.";
} else {
// The testing framework (package:testing) sets the exitCode to `1` if any
// test failed, so we reset it here to indicate that the test runner was
// successful.
exitCode = 0;
}
}