| // Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE.md file. |
| |
| import 'dart:async' show Timer; |
| import 'dart:convert' show jsonEncode; |
| import 'dart:io' show File, Platform, exitCode; |
| import 'dart:isolate' show Isolate, ReceivePort, SendPort; |
| |
| import 'package:args/args.dart' show ArgParser; |
| import 'package:testing/src/chain.dart' show CreateContext, Result, Step; |
| import 'package:testing/src/expectation.dart' show Expectation; |
| import 'package:testing/src/log.dart' show Logger; |
| import 'package:testing/src/run.dart' show runMe; |
| import 'package:testing/src/suite.dart' as testing show Suite; |
| import 'package:testing/src/test_description.dart' show TestDescription; |
| |
| import 'dartdoctest_suite.dart' as dartdoctest show createContext; |
| import 'fasta/expression_suite.dart' as expression show createContext; |
| import 'fasta/incremental_dartino_suite.dart' as incremental_dartino |
| show createContext; |
| import 'fasta/messages_suite.dart' as messages show createContext; |
| import 'fasta/modular_suite.dart' as modular show createContext; |
| import 'fasta/outline_suite.dart' as outline show createContext; |
| import 'fasta/strong_suite.dart' as strong show createContext; |
| import 'fasta/textual_outline_suite.dart' as textual_outline show createContext; |
| import 'fasta/weak_suite.dart' as weak show createContext; |
| import 'incremental_bulk_compiler_smoke_suite.dart' as incremental_bulk_compiler |
| show createContext; |
| import 'incremental_suite.dart' as incremental show createContext; |
| import 'lint_suite.dart' as lint show createContext; |
| import 'outline_extractor_suite.dart' as outline_extractor show createContext; |
| import 'parser_all_suite.dart' as parserAll show createContext; |
| import 'parser_equivalence_suite.dart' as parserEquivalence show createContext; |
| import 'parser_suite.dart' as parser show createContext; |
| import 'spelling_test_not_src_suite.dart' as spelling_not_src |
| show createContext; |
| import 'spelling_test_src_suite.dart' as spelling_src show createContext; |
| |
| const suiteNamePrefix = "pkg/front_end/test"; |
| |
| int getDefaultThreads() { |
| int numberOfWorkers = 1; |
| if (Platform.numberOfProcessors > 2) { |
| numberOfWorkers = Platform.numberOfProcessors - 1; |
| } |
| return numberOfWorkers; |
| } |
| |
| class Options { |
| final String? configurationName; |
| final bool verbose; |
| final bool printFailureLog; |
| final Uri outputDirectory; |
| final String? testFilter; |
| final List<String> environmentOptions; |
| final int shardCount; |
| final int shard; |
| final bool skipTestsThatRequireGit; |
| final bool onlyTestsThatRequireGit; |
| final int numberOfWorkers; |
| |
| Options( |
| this.configurationName, |
| this.verbose, |
| this.printFailureLog, |
| this.outputDirectory, |
| this.testFilter, |
| this.environmentOptions, { |
| required this.shardCount, |
| required this.shard, |
| required this.skipTestsThatRequireGit, |
| required this.onlyTestsThatRequireGit, |
| required this.numberOfWorkers, |
| }); |
| |
| static Options parse(List<String> args) { |
| var parser = new ArgParser() |
| ..addOption("named-configuration", |
| abbr: "n", |
| help: "configuration name to use for emitting json result files") |
| ..addOption("output-directory", |
| help: "directory to which results.json and logs.json are written") |
| ..addFlag("verbose", |
| abbr: "v", help: "print additional information", defaultsTo: false) |
| ..addFlag("print", |
| abbr: "p", help: "print failure logs", defaultsTo: false) |
| ..addMultiOption('environment', |
| abbr: 'D', help: "environment options for the test suite") |
| ..addOption("tasks", |
| abbr: "j", |
| help: "The number of parallel tasks to run.", |
| defaultsTo: "${getDefaultThreads()}") |
| ..addOption("shards", help: "Number of shards", defaultsTo: "1") |
| ..addOption("shard", help: "Which shard to run", defaultsTo: "1") |
| ..addFlag("skipTestsThatRequireGit", |
| help: "Whether to skip tests that require git to run", |
| defaultsTo: false) |
| ..addFlag("onlyTestsThatRequireGit", |
| help: "Whether to only run tests that require git", |
| defaultsTo: false); |
| var parsedOptions = parser.parse(args); |
| String outputPath = parsedOptions["output-directory"] ?? "."; |
| Uri outputDirectory = Uri.base.resolveUri(Uri.directory(outputPath)); |
| |
| bool verbose = parsedOptions["verbose"]; |
| |
| String? filter; |
| if (parsedOptions.rest.length == 1) { |
| filter = parsedOptions.rest.single; |
| if (filter.startsWith("$suiteNamePrefix/")) { |
| filter = filter.substring(suiteNamePrefix.length + 1); |
| } |
| } |
| String tasksString = parsedOptions["tasks"]; |
| int? tasks = int.tryParse(tasksString); |
| if (tasks == null || tasks < 1) { |
| throw "--tasks (-j) has to be an integer >= 1"; |
| } |
| |
| String shardsString = parsedOptions["shards"]; |
| int? shardCount = int.tryParse(shardsString); |
| if (shardCount == null || shardCount < 1) { |
| throw "--shards has to be an integer >= 1"; |
| } |
| String shardString = parsedOptions["shard"]; |
| int? shard = int.tryParse(shardString); |
| if (shard == null || shard < 1 || shard > shardCount) { |
| throw "--shard has to be an integer >= 1 (and <= --shards)"; |
| } |
| bool skipTestsThatRequireGit = parsedOptions["skipTestsThatRequireGit"]; |
| bool onlyTestsThatRequireGit = parsedOptions["onlyTestsThatRequireGit"]; |
| if (skipTestsThatRequireGit && onlyTestsThatRequireGit) { |
| throw "Only one of --skipTestsThatRequireGit and " |
| "--onlyTestsThatRequireGit can be provided."; |
| } |
| |
| if (verbose) { |
| print("NOTE: Created with options\n " |
| "${parsedOptions["named-configuration"]},\n " |
| "${verbose},\n " |
| "${parsedOptions["print"]},\n " |
| "${outputDirectory},\n " |
| "${filter},\n " |
| "${parsedOptions['environment']},\n " |
| "shardCount: ${shardCount},\n " |
| "shard: ${shard - 1 /* make it 0-indexed */},\n " |
| "onlyTestsThatRequireGit: ${onlyTestsThatRequireGit},\n " |
| "skipTestsThatRequireGit: ${skipTestsThatRequireGit},\n " |
| "numberOfWorkers: ${tasks}"); |
| } |
| |
| return Options( |
| parsedOptions["named-configuration"], |
| verbose, |
| parsedOptions["print"], |
| outputDirectory, |
| filter, |
| parsedOptions['environment'], |
| shardCount: shardCount, |
| shard: shard - 1 /* make it 0-indexed */, |
| onlyTestsThatRequireGit: onlyTestsThatRequireGit, |
| skipTestsThatRequireGit: skipTestsThatRequireGit, |
| numberOfWorkers: tasks, |
| ); |
| } |
| } |
| |
| class ResultLogger implements Logger { |
| final String prefix; |
| final bool verbose; |
| final bool printFailureLog; |
| final SendPort resultsPort; |
| final SendPort logsPort; |
| final Map<String, Stopwatch> stopwatches = {}; |
| final String? configurationName; |
| final Set<String> seenTests = {}; |
| bool gotFrameworkError = false; |
| |
| ResultLogger(this.prefix, this.resultsPort, this.logsPort, this.verbose, |
| this.printFailureLog, this.configurationName); |
| |
| String getTestName(TestDescription description) { |
| return "$prefix/${description.shortName}"; |
| } |
| |
| @override |
| void logMessage(Object message) {} |
| |
| @override |
| void logNumberedLines(String text) {} |
| |
| @override |
| void logProgress(String message) {} |
| |
| @override |
| void logStepComplete(int completed, int failed, int total, |
| testing.Suite suite, TestDescription description, Step step) {} |
| |
| @override |
| void logStepStart(int completed, int failed, int total, testing.Suite suite, |
| TestDescription description, Step step) {} |
| |
| @override |
| void logSuiteStarted(testing.Suite suite) {} |
| |
| @override |
| void logSuiteComplete(testing.Suite suite) {} |
| |
| void handleTestResult( |
| testing.Suite suite, |
| TestDescription testDescription, |
| Result result, |
| String fullSuiteName, |
| bool matchedExpectations, |
| Set<Expectation> expectedOutcomes) { |
| String testName = getTestName(testDescription); |
| String suiteName = "pkg"; |
| String shortTestName = testName.substring(suiteName.length + 1); |
| resultsPort.send(jsonEncode({ |
| "name": testName, |
| "configuration": configurationName, |
| "suite": suiteName, |
| "test_name": shortTestName, |
| "time_ms": stopwatches[testName]!.elapsedMilliseconds, |
| "expected": "Pass", |
| "result": matchedExpectations ? "Pass" : "Fail", |
| "matches": matchedExpectations, |
| })); |
| if (!matchedExpectations) { |
| StringBuffer sb = new StringBuffer(); |
| if (printFailureLog) { |
| String outcome = "${result.outcome}"; |
| sb.write("FAILED: $testName: $outcome"); |
| } |
| sb.write(result.log); |
| if (result.error != null) { |
| sb.write("\n\n${result.error}"); |
| } |
| if (result.trace != null) { |
| sb.write("\n\n${result.trace}"); |
| } |
| sb.write("\n\nTo re-run this test, run:"); |
| sb.write("\n\n dart pkg/front_end/test/unit_test_suites.dart -p " |
| "$testName"); |
| if (result.autoFixCommand != null) { |
| sb.write("\n\nTo automatically update the test expectations, run:"); |
| sb.write("\n\n dart pkg/front_end/test/unit_test_suites.dart -p " |
| "$testName -D${result.autoFixCommand}"); |
| if (result.canBeFixWithUpdateExpectations) { |
| sb.write('\n\nTo update test expectations for all tests at once, ' |
| 'run:'); |
| sb.write('\n\n dart pkg/front_end/tool/update_expectations.dart'); |
| sb.write('\n\nNote that this takes a long time and should only be ' |
| 'used when many tests need updating.\n'); |
| } |
| } |
| if (result.outcome == Expectation.Pass) { |
| String expectedString = |
| expectedOutcomes.map((e) => e.toString()).join(", "); |
| sb.write("\n\nThe test passed, but wasn't expected to. " |
| "You should update the status file for this test." |
| "\nThere's a status entry looking something like" |
| "\n\n ${testDescription.shortName}: ${expectedString}" |
| "\n\nwhich should be removed." |
| "\n\nThe status file is ${suite.statusFile}."); |
| } else if (result.autoFixCommand == null) { |
| String expectedString = |
| expectedOutcomes.map((e) => e.toString()).join(", "); |
| sb.write("\n\nThe test has outcome ${result.outcome}, " |
| "but was expected to have outcome(s) ${expectedOutcomes}. " |
| "You might have to update the status file to the new outcome" |
| "\nThere's a status entry looking something like" |
| "\n\n ${testDescription.shortName}: ${expectedString}" |
| "\n\nwhich should be updated." |
| "\n\nThe status file is ${suite.statusFile}."); |
| } |
| String failureLog = sb.toString(); |
| String outcome = "${result.outcome}"; |
| logsPort.send(jsonEncode({ |
| "name": testName, |
| "configuration": configurationName, |
| "result": outcome, |
| "log": failureLog, |
| })); |
| if (printFailureLog) { |
| print(failureLog); |
| } |
| } |
| if (verbose) { |
| String result = matchedExpectations ? "PASS" : "FAIL"; |
| print("${testName}: ${result}"); |
| } |
| } |
| |
| @override |
| void logTestStart(int completed, int failed, int total, testing.Suite suite, |
| TestDescription description) { |
| String name = getTestName(description); |
| stopwatches[name] = Stopwatch()..start(); |
| } |
| |
| @override |
| void logTestComplete(int completed, int failed, int total, |
| testing.Suite suite, TestDescription description) {} |
| |
| @override |
| void logUncaughtError(error, StackTrace stackTrace) {} |
| |
| @override |
| void logExpectedResult(testing.Suite suite, TestDescription description, |
| Result result, Set<Expectation> expectedOutcomes) { |
| handleTestResult( |
| suite, description, result, prefix, true, expectedOutcomes); |
| } |
| |
| @override |
| void logUnexpectedResult(testing.Suite suite, TestDescription description, |
| Result result, Set<Expectation> expectedOutcomes) { |
| // The test framework (pkg/testing) calls the logger with an unexpected |
| // results a second time to create a summary. We ignore the second call |
| // here. |
| String testName = getTestName(description); |
| if (seenTests.contains(testName)) return; |
| seenTests.add(testName); |
| handleTestResult( |
| suite, description, result, prefix, false, expectedOutcomes); |
| } |
| |
| @override |
| void noticeFrameworkCatchError(error, StackTrace stackTrace) { |
| gotFrameworkError = true; |
| } |
| } |
| |
| class Suite { |
| final CreateContext createContext; |
| final String testingRootPath; |
| final String? path; |
| final int shardCount; |
| final String prefix; |
| final bool requiresGit; |
| |
| const Suite( |
| this.prefix, |
| this.createContext, |
| this.testingRootPath, { |
| required this.shardCount, |
| this.path, |
| this.requiresGit: false, |
| }); |
| } |
| |
| const List<Suite> suites = [ |
| const Suite( |
| "dartdoctest", |
| dartdoctest.createContext, |
| "../testing.json", |
| shardCount: 1, |
| ), |
| const Suite( |
| "fasta/expression", |
| expression.createContext, |
| "../../testing.json", |
| shardCount: 1, |
| ), |
| const Suite( |
| "fasta/outline", |
| outline.createContext, |
| "../../testing.json", |
| shardCount: 2, |
| ), |
| const Suite( |
| "fasta/incremental_dartino", |
| incremental_dartino.createContext, |
| "../../testing.json", |
| shardCount: 1, |
| ), |
| const Suite( |
| "fasta/messages", |
| messages.createContext, |
| "../../testing.json", |
| shardCount: 1, |
| requiresGit: true, |
| ), |
| const Suite( |
| "fasta/strong", |
| strong.createContext, |
| "../../testing.json", |
| path: "fasta/strong_suite.dart", |
| shardCount: 2, |
| ), |
| const Suite( |
| "incremental_bulk_compiler_smoke", |
| incremental_bulk_compiler.createContext, |
| "../testing.json", |
| shardCount: 1, |
| ), |
| const Suite( |
| "incremental", |
| incremental.createContext, |
| "../testing.json", |
| shardCount: 2, |
| ), |
| const Suite( |
| "lint", |
| lint.createContext, |
| "../testing.json", |
| shardCount: 1, |
| requiresGit: true, |
| ), |
| const Suite( |
| "parser", |
| parser.createContext, |
| "../testing.json", |
| shardCount: 1, |
| ), |
| const Suite( |
| "parser_equivalence", |
| parserEquivalence.createContext, |
| "../testing.json", |
| shardCount: 1, |
| ), |
| const Suite( |
| "parser_all", |
| parserAll.createContext, |
| "../testing.json", |
| shardCount: 4, |
| requiresGit: |
| true /* technically not true, but tests *many* more files |
| than in test_matrix.json file set */ |
| , |
| ), |
| const Suite( |
| "spelling_test_not_src", |
| spelling_not_src.createContext, |
| "../testing.json", |
| shardCount: 1, |
| requiresGit: true, |
| ), |
| const Suite( |
| "spelling_test_src", |
| spelling_src.createContext, |
| "../testing.json", |
| shardCount: 1, |
| requiresGit: true, |
| ), |
| const Suite( |
| "fasta/modular", |
| modular.createContext, |
| "../../testing.json", |
| path: "fasta/modular_suite.dart", |
| shardCount: 4, |
| ), |
| const Suite( |
| "fasta/weak", |
| weak.createContext, |
| "../../testing.json", |
| path: "fasta/weak_suite.dart", |
| shardCount: 10, |
| ), |
| const Suite( |
| "fasta/textual_outline", |
| textual_outline.createContext, |
| "../../testing.json", |
| shardCount: 1, |
| ), |
| const Suite( |
| "outline_extractor", |
| outline_extractor.createContext, |
| "../testing.json", |
| shardCount: 1, |
| ), |
| ]; |
| |
| const Duration timeoutDuration = Duration(minutes: 30); |
| |
| class SuiteConfiguration { |
| final Suite suite; |
| final SendPort resultsPort; |
| final SendPort logsPort; |
| final bool verbose; |
| final bool printFailureLog; |
| final String? configurationName; |
| final String? testFilter; |
| final List<String> environmentOptions; |
| final int shard; |
| |
| const SuiteConfiguration( |
| this.suite, |
| this.resultsPort, |
| this.logsPort, |
| this.verbose, |
| this.printFailureLog, |
| this.configurationName, |
| this.testFilter, |
| this.environmentOptions, |
| this.shard, |
| ); |
| } |
| |
| Future<void> runSuite(SuiteConfiguration configuration) async { |
| Suite suite = configuration.suite; |
| String name = suite.prefix; |
| String fullSuiteName = "$suiteNamePrefix/$name"; |
| Uri suiteUri = Platform.script.resolve(suite.path ?? "${name}_suite.dart"); |
| if (!new File.fromUri(suiteUri).existsSync()) { |
| throw "File doesn't exist: $suiteUri"; |
| } |
| ResultLogger logger = ResultLogger( |
| fullSuiteName, |
| configuration.resultsPort, |
| configuration.logsPort, |
| configuration.verbose, |
| configuration.printFailureLog, |
| configuration.configurationName); |
| await runMe( |
| <String>[ |
| if (configuration.testFilter != null) configuration.testFilter!, |
| for (String option in configuration.environmentOptions) '-D${option}', |
| ], |
| suite.createContext, |
| me: suiteUri, |
| configurationPath: suite.testingRootPath, |
| logger: logger, |
| shards: suite.shardCount, |
| shard: configuration.shard, |
| ); |
| if (logger.gotFrameworkError) { |
| throw "Got framework error!"; |
| } |
| } |
| |
| Future<void> writeLinesToFile(Uri uri, List<String> lines) async { |
| await File.fromUri(uri).writeAsString(lines.map((line) => "$line\n").join()); |
| } |
| |
| Future<void> main([List<String> arguments = const <String>[]]) async { |
| Stopwatch totalRuntime = new Stopwatch()..start(); |
| |
| List<String> results = []; |
| List<String> logs = []; |
| Options options = Options.parse(arguments); |
| ReceivePort resultsPort = new ReceivePort() |
| ..listen((resultEntry) => results.add(resultEntry)); |
| ReceivePort logsPort = new ReceivePort() |
| ..listen((logEntry) => logs.add(logEntry)); |
| List<Future<bool>> futures = []; |
| |
| if (options.verbose) { |
| print("NOTE: Willing to run with ${options.numberOfWorkers} 'workers'"); |
| print(""); |
| } |
| |
| int numberOfFreeWorkers = options.numberOfWorkers; |
| // Run test suites and record the results and possible failure logs. |
| int chunkNum = 0; |
| for (Suite suite in suites) { |
| if (options.onlyTestsThatRequireGit && !suite.requiresGit) continue; |
| if (options.skipTestsThatRequireGit && suite.requiresGit) continue; |
| String prefix = suite.prefix; |
| String? filter = options.testFilter; |
| if (filter != null) { |
| // Skip suites that are not hit by the test filter, is there is one. |
| if (!filter.startsWith(prefix)) { |
| continue; |
| } |
| // Remove the 'fasta/' from filters, if there, because it is not used |
| // in the name defined in testing.json. |
| if (filter.startsWith("fasta/")) { |
| filter = filter.substring("fasta/".length); |
| } |
| } |
| for (int shard = 0; shard < suite.shardCount; shard++) { |
| if (chunkNum++ % options.shardCount != options.shard) continue; |
| |
| while (numberOfFreeWorkers <= 0) { |
| // This might not be great design, but it'll work fine. |
| await Future.delayed(const Duration(milliseconds: 50)); |
| } |
| numberOfFreeWorkers--; |
| // Start the test suite in a new isolate. |
| ReceivePort exitPort = new ReceivePort(); |
| ReceivePort errorPort = new ReceivePort(); |
| SuiteConfiguration configuration = new SuiteConfiguration( |
| suite, |
| resultsPort.sendPort, |
| logsPort.sendPort, |
| options.verbose, |
| options.printFailureLog, |
| options.configurationName, |
| filter, |
| options.environmentOptions, |
| shard); |
| Future<bool> future = new Future<bool>(() async { |
| try { |
| Stopwatch stopwatch = new Stopwatch()..start(); |
| String naming = "$prefix"; |
| if (suite.shardCount > 1) { |
| naming += " (${shard + 1} of ${suite.shardCount})"; |
| } |
| print("Running suite $naming"); |
| Isolate isolate = await Isolate.spawn<SuiteConfiguration>( |
| runSuite, configuration, |
| onExit: exitPort.sendPort, onError: errorPort.sendPort); |
| bool timedOutOrCrash = false; |
| Timer timer = new Timer(timeoutDuration, () { |
| timedOutOrCrash = true; |
| print("Suite $naming timed out after " |
| "${timeoutDuration.inMilliseconds}ms"); |
| isolate.kill(priority: Isolate.immediate); |
| }); |
| await exitPort.first; |
| errorPort.close(); |
| List<dynamic> allErrors = await errorPort.toList(); |
| bool gotError = allErrors.isNotEmpty; |
| if (gotError) { |
| print("Suite $naming encountered ${allErrors.length} error(s)."); |
| print("Errors:"); |
| for (int i = 0; i < allErrors.length; i++) { |
| print("-----------"); |
| print("Error #$i:"); |
| print(allErrors[i]); |
| } |
| print("-----------"); |
| timedOutOrCrash = true; |
| } |
| timer.cancel(); |
| int seconds = stopwatch.elapsedMilliseconds ~/ 1000; |
| if (!timedOutOrCrash) { |
| print("Suite $naming finished (took ${seconds} seconds)"); |
| } else { |
| print("Suite $naming finished badly (see above) " |
| "(took ${seconds} seconds)"); |
| } |
| return timedOutOrCrash; |
| } finally { |
| numberOfFreeWorkers++; |
| } |
| }); |
| futures.add(future); |
| } |
| } |
| // Wait for isolates to terminate and clean up. |
| Iterable<bool> timeoutsOrCrashes = await Future.wait(futures); |
| resultsPort.close(); |
| logsPort.close(); |
| // Write results.json and logs.json. |
| Uri resultJsonUri = options.outputDirectory.resolve("results.json"); |
| Uri logsJsonUri = options.outputDirectory.resolve("logs.json"); |
| await writeLinesToFile(resultJsonUri, results); |
| await writeLinesToFile(logsJsonUri, logs); |
| print("Log files written to ${resultJsonUri.toFilePath()} and" |
| " ${logsJsonUri.toFilePath()}"); |
| print("Entire run took ${totalRuntime.elapsed}."); |
| // Return with exit code 1 if at least one suite timed out. |
| bool timedOutOrCrashed = timeoutsOrCrashes.any((timeout) => timeout); |
| if (timedOutOrCrashed) { |
| throw "Crashed or timed out. Check stdout for more details."; |
| } else { |
| // The testing framework (package:testing) sets the exitCode to `1` if any |
| // test failed, so we reset it here to indicate that the test runner was |
| // successful. |
| exitCode = 0; |
| } |
| } |