speed up tests by using a precompiled test runner (#1605)
Adds a new `precompileTestExecutable` method which you must call before calling `runTest`. It should be invoked in a top level `setUpAll` method, and will clean up its output in `tearDownAll`.
This should significantly speed up tests using `runTest` more than once.
diff --git a/pkgs/test/test/io.dart b/pkgs/test/test/io.dart
index 1820980..af9bfd4 100644
--- a/pkgs/test/test/io.dart
+++ b/pkgs/test/test/io.dart
@@ -67,7 +67,39 @@
StreamMatcher containsInOrder(Iterable<String> strings) =>
emitsInOrder(strings.map((string) => emitsThrough(contains(string))));
+/// Lazily compile the test package to kernel and re-use that, initialized with
+/// [precompileTestExecutable].
+String? _testExecutablePath;
+
+/// Must be invoked before any call to [runTests], should be invoked in a top
+/// level `setUpAll` for best caching results.
+Future<void> precompileTestExecutable() async {
+ if (_testExecutablePath != null) {
+ throw StateError('Test executable already precompiled');
+ }
+ var tmpDirectory = await Directory.systemTemp.createTemp('test');
+ var precompiledPath = p.join(tmpDirectory.path, 'test_runner.dill');
+ var result = await Process.run(Platform.executable, [
+ 'compile',
+ 'kernel',
+ p.url.join(await packageDir, 'bin', 'test.dart'),
+ '-o',
+ precompiledPath,
+ ]);
+ if (result.exitCode != 0) {
+ throw StateError(
+ 'Failed to compile test runner:\n${result.stdout}\n${result.stderr}');
+ }
+
+ addTearDown(() async {
+ await tmpDirectory.delete(recursive: true);
+ });
+ _testExecutablePath = precompiledPath;
+}
+
/// Runs the test executable with the package root set properly.
+///
+/// You must invoke [precompileTestExecutable] before invoking this function.
Future<TestProcess> runTest(Iterable<String> args,
{String? reporter,
String? fileReporter,
@@ -77,10 +109,15 @@
String? packageConfig,
Iterable<String>? vmArgs}) async {
concurrency ??= 1;
+ var testExecutablePath = _testExecutablePath;
+ if (testExecutablePath == null) {
+ throw StateError(
+ 'You must call `precompileTestExecutable` before calling `runTest`');
+ }
var allArgs = [
...?vmArgs,
- Uri.file(p.url.join(await packageDir, 'bin', 'test.dart')).toString(),
+ testExecutablePath,
'--concurrency=$concurrency',
if (reporter != null) '--reporter=$reporter',
if (fileReporter != null) '--file-reporter=$fileReporter',
diff --git a/pkgs/test/test/runner/browser/chrome_test.dart b/pkgs/test/test/runner/browser/chrome_test.dart
index b5ac51e..fb556e8 100644
--- a/pkgs/test/test/runner/browser/chrome_test.dart
+++ b/pkgs/test/test/runner/browser/chrome_test.dart
@@ -15,6 +15,8 @@
import 'code_server.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('starts Chrome with the given URL', () async {
var server = await CodeServer.start();
diff --git a/pkgs/test/test/runner/browser/compact_reporter_test.dart b/pkgs/test/test/runner/browser/compact_reporter_test.dart
index 4a65f92..c8a14fc 100644
--- a/pkgs/test/test/runner/browser/compact_reporter_test.dart
+++ b/pkgs/test/test/runner/browser/compact_reporter_test.dart
@@ -10,6 +10,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('prints the platform name when running on multiple platforms', () async {
await d.file('test.dart', '''
import 'dart:async';
diff --git a/pkgs/test/test/runner/browser/expanded_reporter_test.dart b/pkgs/test/test/runner/browser/expanded_reporter_test.dart
index c334843..21b1bc4 100644
--- a/pkgs/test/test/runner/browser/expanded_reporter_test.dart
+++ b/pkgs/test/test/runner/browser/expanded_reporter_test.dart
@@ -11,6 +11,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('prints the platform name when running on multiple platforms', () async {
await d.file('test.dart', '''
import 'dart:async';
diff --git a/pkgs/test/test/runner/browser/firefox_test.dart b/pkgs/test/test/runner/browser/firefox_test.dart
index 748eafe..d1ac035 100644
--- a/pkgs/test/test/runner/browser/firefox_test.dart
+++ b/pkgs/test/test/runner/browser/firefox_test.dart
@@ -16,6 +16,8 @@
import 'code_server.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('starts Firefox with the given URL', () async {
var server = await CodeServer.start();
diff --git a/pkgs/test/test/runner/browser/internet_explorer_test.dart b/pkgs/test/test/runner/browser/internet_explorer_test.dart
index e240755..a0fadbf 100644
--- a/pkgs/test/test/runner/browser/internet_explorer_test.dart
+++ b/pkgs/test/test/runner/browser/internet_explorer_test.dart
@@ -16,6 +16,8 @@
import 'code_server.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('starts IE with the given URL', () async {
var server = await CodeServer.start();
diff --git a/pkgs/test/test/runner/browser/runner_test.dart b/pkgs/test/test/runner/browser/runner_test.dart
index ccd12ce..c9e2a44 100644
--- a/pkgs/test/test/runner/browser/runner_test.dart
+++ b/pkgs/test/test/runner/browser/runner_test.dart
@@ -28,6 +28,8 @@
''';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('fails gracefully if', () {
test('a test file fails to compile', () async {
await d.file('test.dart', 'invalid Dart file').create();
diff --git a/pkgs/test/test/runner/browser/safari_test.dart b/pkgs/test/test/runner/browser/safari_test.dart
index e1f5a1a..f45f5e6 100644
--- a/pkgs/test/test/runner/browser/safari_test.dart
+++ b/pkgs/test/test/runner/browser/safari_test.dart
@@ -16,6 +16,8 @@
import 'code_server.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('starts Safari with the given URL', () async {
var server = await CodeServer.start();
diff --git a/pkgs/test/test/runner/compact_reporter_test.dart b/pkgs/test/test/runner/compact_reporter_test.dart
index 4e21916..92b5c67 100644
--- a/pkgs/test/test/runner/compact_reporter_test.dart
+++ b/pkgs/test/test/runner/compact_reporter_test.dart
@@ -13,6 +13,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('reports when no tests are run', () async {
await d.file('test.dart', 'void main() {}').create();
diff --git a/pkgs/test/test/runner/configuration/custom_platform_test.dart b/pkgs/test/test/runner/configuration/custom_platform_test.dart
index 050a32c..39a6bbd 100644
--- a/pkgs/test/test/runner/configuration/custom_platform_test.dart
+++ b/pkgs/test/test/runner/configuration/custom_platform_test.dart
@@ -17,6 +17,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
setUp(() async {
await d.file('test.dart', '''
import 'package:test/test.dart';
diff --git a/pkgs/test/test/runner/configuration/duplicate_names_test.dart b/pkgs/test/test/runner/configuration/duplicate_names_test.dart
index 8e5fe9e..a6f6cc4 100644
--- a/pkgs/test/test/runner/configuration/duplicate_names_test.dart
+++ b/pkgs/test/test/runner/configuration/duplicate_names_test.dart
@@ -14,6 +14,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('duplicate names', () {
group('can be disabled for', () {
for (var function in ['group', 'test']) {
diff --git a/pkgs/test/test/runner/configuration/global_test.dart b/pkgs/test/test/runner/configuration/global_test.dart
index 1ebd0cf..dce108b 100644
--- a/pkgs/test/test/runner/configuration/global_test.dart
+++ b/pkgs/test/test/runner/configuration/global_test.dart
@@ -13,6 +13,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('ignores an empty file', () async {
await d.file('global_test.yaml', '').create();
diff --git a/pkgs/test/test/runner/configuration/platform_test.dart b/pkgs/test/test/runner/configuration/platform_test.dart
index d9bcede..a111801 100644
--- a/pkgs/test/test/runner/configuration/platform_test.dart
+++ b/pkgs/test/test/runner/configuration/platform_test.dart
@@ -14,6 +14,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('on_platform', () {
test('applies platform-specific configuration to matching tests', () async {
await d
diff --git a/pkgs/test/test/runner/configuration/presets_test.dart b/pkgs/test/test/runner/configuration/presets_test.dart
index e802a87..1d3b34a 100644
--- a/pkgs/test/test/runner/configuration/presets_test.dart
+++ b/pkgs/test/test/runner/configuration/presets_test.dart
@@ -14,6 +14,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('presets', () {
test("don't do anything by default", () async {
await d
diff --git a/pkgs/test/test/runner/configuration/randomize_order_test.dart b/pkgs/test/test/runner/configuration/randomize_order_test.dart
index adb853e..7d39aa3 100644
--- a/pkgs/test/test/runner/configuration/randomize_order_test.dart
+++ b/pkgs/test/test/runner/configuration/randomize_order_test.dart
@@ -12,6 +12,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('shuffles test order when passed a seed', () async {
await d.file('test.dart', '''
import 'package:test/test.dart';
@@ -176,7 +178,7 @@
test("test 2.2", () {});
test("test 2.3", () {});
test("test 2.4", () {});
- });
+ });
}
''').create();
diff --git a/pkgs/test/test/runner/configuration/tags_test.dart b/pkgs/test/test/runner/configuration/tags_test.dart
index c5fc8b6..208f119 100644
--- a/pkgs/test/test/runner/configuration/tags_test.dart
+++ b/pkgs/test/test/runner/configuration/tags_test.dart
@@ -14,6 +14,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('adds the specified tags', () async {
await d
.file(
diff --git a/pkgs/test/test/runner/configuration/top_level_error_test.dart b/pkgs/test/test/runner/configuration/top_level_error_test.dart
index eb08350..e4fd130 100644
--- a/pkgs/test/test/runner/configuration/top_level_error_test.dart
+++ b/pkgs/test/test/runner/configuration/top_level_error_test.dart
@@ -13,6 +13,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('rejects an invalid fold_stack_frames', () async {
await d
.file('dart_test.yaml', jsonEncode({'fold_stack_frames': 'flup'}))
diff --git a/pkgs/test/test/runner/configuration/top_level_test.dart b/pkgs/test/test/runner/configuration/top_level_test.dart
index d1031b5..2670c7a 100644
--- a/pkgs/test/test/runner/configuration/top_level_test.dart
+++ b/pkgs/test/test/runner/configuration/top_level_test.dart
@@ -15,6 +15,8 @@
import '../../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('ignores an empty file', () async {
await d.file('dart_test.yaml', '').create();
diff --git a/pkgs/test/test/runner/coverage_test.dart b/pkgs/test/test/runner/coverage_test.dart
index 9298be3..a9005db 100644
--- a/pkgs/test/test/runner/coverage_test.dart
+++ b/pkgs/test/test/runner/coverage_test.dart
@@ -16,6 +16,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('with the --coverage flag,', () {
late Directory coverageDirectory;
diff --git a/pkgs/test/test/runner/data_isolate_strategy_test.dart b/pkgs/test/test/runner/data_isolate_strategy_test.dart
index 7063d50..5bae348 100644
--- a/pkgs/test/test/runner/data_isolate_strategy_test.dart
+++ b/pkgs/test/test/runner/data_isolate_strategy_test.dart
@@ -18,6 +18,7 @@
late PackageConfig currentPackageConfig;
setUpAll(() async {
+ await precompileTestExecutable();
currentPackageConfig =
await loadPackageConfigUri((await Isolate.packageConfig)!);
});
diff --git a/pkgs/test/test/runner/expanded_reporter_test.dart b/pkgs/test/test/runner/expanded_reporter_test.dart
index 54c0549..b14b676 100644
--- a/pkgs/test/test/runner/expanded_reporter_test.dart
+++ b/pkgs/test/test/runner/expanded_reporter_test.dart
@@ -13,6 +13,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('reports when no tests are run', () async {
await d.file('test.dart', 'void main() {}').create();
diff --git a/pkgs/test/test/runner/hybrid_test.dart b/pkgs/test/test/runner/hybrid_test.dart
index 4e66ce6..ccceb54 100644
--- a/pkgs/test/test/runner/hybrid_test.dart
+++ b/pkgs/test/test/runner/hybrid_test.dart
@@ -17,6 +17,7 @@
void main() {
late String packageRoot;
setUpAll(() async {
+ await precompileTestExecutable();
packageRoot = p.absolute(p.dirname(p
.fromUri(await Isolate.resolvePackageUri(Uri.parse('package:test/')))));
});
diff --git a/pkgs/test/test/runner/json_file_reporter_test.dart b/pkgs/test/test/runner/json_file_reporter_test.dart
index 59b5694..5cc3684 100644
--- a/pkgs/test/test/runner/json_file_reporter_test.dart
+++ b/pkgs/test/test/runner/json_file_reporter_test.dart
@@ -18,6 +18,8 @@
import 'json_reporter_utils.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('runs successful tests with a stdout reporter and file reporter', () {
return _expectReports('''
test('success 1', () {});
diff --git a/pkgs/test/test/runner/json_reporter_test.dart b/pkgs/test/test/runner/json_reporter_test.dart
index e0bd3a9..ede34de 100644
--- a/pkgs/test/test/runner/json_reporter_test.dart
+++ b/pkgs/test/test/runner/json_reporter_test.dart
@@ -15,6 +15,8 @@
import 'json_reporter_utils.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('runs several successful tests and reports when each completes', () {
return _expectReport('''
test('success 1', () {});
diff --git a/pkgs/test/test/runner/name_test.dart b/pkgs/test/test/runner/name_test.dart
index 447ecfa..301b21e 100644
--- a/pkgs/test/test/runner/name_test.dart
+++ b/pkgs/test/test/runner/name_test.dart
@@ -12,6 +12,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('with test.dart?name="name" query', () {
test('selects tests with matching names', () async {
await d.file('test.dart', '''
diff --git a/pkgs/test/test/runner/node/runner_test.dart b/pkgs/test/test/runner/node/runner_test.dart
index 149f951..b5d02c1 100644
--- a/pkgs/test/test/runner/node/runner_test.dart
+++ b/pkgs/test/test/runner/node/runner_test.dart
@@ -29,6 +29,8 @@
''';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('fails gracefully if', () {
test('a test file fails to compile', () async {
await d.file('test.dart', 'invalid Dart file').create();
diff --git a/pkgs/test/test/runner/pause_after_load_test.dart b/pkgs/test/test/runner/pause_after_load_test.dart
index 2dd4704..88576aa 100644
--- a/pkgs/test/test/runner/pause_after_load_test.dart
+++ b/pkgs/test/test/runner/pause_after_load_test.dart
@@ -13,6 +13,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('pauses the test runner for each file until the user presses enter',
() async {
await d.file('test1.dart', '''
diff --git a/pkgs/test/test/runner/precompiled_test.dart b/pkgs/test/test/runner/precompiled_test.dart
index 72022fa..1c0ec2b 100644
--- a/pkgs/test/test/runner/precompiled_test.dart
+++ b/pkgs/test/test/runner/precompiled_test.dart
@@ -20,6 +20,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('browser tests', () {
setUp(() async {
await d.file('to_precompile.dart', '''
diff --git a/pkgs/test/test/runner/pub_serve_test.dart b/pkgs/test/test/runner/pub_serve_test.dart
index 744385c..35d2578 100644
--- a/pkgs/test/test/runner/pub_serve_test.dart
+++ b/pkgs/test/test/runner/pub_serve_test.dart
@@ -19,6 +19,8 @@
String get _pubServeArg => '--pub-serve=$pubServePort';
void main() {
+ setUpAll(precompileTestExecutable);
+
setUp(() async {
await d.file('pubspec.yaml', '''
name: myapp
diff --git a/pkgs/test/test/runner/retry_test.dart b/pkgs/test/test/runner/retry_test.dart
index 8a3585f..519ce9a 100644
--- a/pkgs/test/test/runner/retry_test.dart
+++ b/pkgs/test/test/runner/retry_test.dart
@@ -10,6 +10,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('respects --no-retry flag with retry option', () async {
await d.file('test.dart', '''
import 'dart:async';
diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart
index 75ca350..2d1eadf 100644
--- a/pkgs/test/test/runner/runner_test.dart
+++ b/pkgs/test/test/runner/runner_test.dart
@@ -123,6 +123,8 @@
', node]';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('prints help information', () async {
var test = await runTest(['--help']);
expectStdoutEquals(test, '''
diff --git a/pkgs/test/test/runner/set_up_all_test.dart b/pkgs/test/test/runner/set_up_all_test.dart
index e029019..1e280c6 100644
--- a/pkgs/test/test/runner/set_up_all_test.dart
+++ b/pkgs/test/test/runner/set_up_all_test.dart
@@ -11,6 +11,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('an error causes the run to fail', () async {
await d.file('test.dart', r'''
import 'package:test/test.dart';
diff --git a/pkgs/test/test/runner/shard_test.dart b/pkgs/test/test/runner/shard_test.dart
index 311b91e..5f1be9c 100644
--- a/pkgs/test/test/runner/shard_test.dart
+++ b/pkgs/test/test/runner/shard_test.dart
@@ -12,6 +12,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('divides all the tests among the available shards', () async {
await d.file('test.dart', '''
import 'package:test/test.dart';
diff --git a/pkgs/test/test/runner/signal_test.dart b/pkgs/test/test/runner/signal_test.dart
index d141329..f456494 100644
--- a/pkgs/test/test/runner/signal_test.dart
+++ b/pkgs/test/test/runner/signal_test.dart
@@ -23,6 +23,8 @@
// represent a best effort to kill the test runner at certain times during its
// execution.
void main() {
+ setUpAll(precompileTestExecutable);
+
setUp(() => d.dir('tmp').create());
group('during loading,', () {
diff --git a/pkgs/test/test/runner/skip_expect_test.dart b/pkgs/test/test/runner/skip_expect_test.dart
index e728827..6d51fac 100644
--- a/pkgs/test/test/runner/skip_expect_test.dart
+++ b/pkgs/test/test/runner/skip_expect_test.dart
@@ -11,6 +11,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
group('a skipped expect', () {
test('marks the test as skipped', () async {
await d.file('test.dart', '''
diff --git a/pkgs/test/test/runner/solo_test.dart b/pkgs/test/test/runner/solo_test.dart
index 4bdb199..508ac67 100644
--- a/pkgs/test/test/runner/solo_test.dart
+++ b/pkgs/test/test/runner/solo_test.dart
@@ -10,6 +10,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('only runs the tests marked as solo', () async {
await d.file('test.dart', '''
import 'dart:async';
diff --git a/pkgs/test/test/runner/tag_test.dart b/pkgs/test/test/runner/tag_test.dart
index d7a15c9..f2f6d19 100644
--- a/pkgs/test/test/runner/tag_test.dart
+++ b/pkgs/test/test/runner/tag_test.dart
@@ -11,6 +11,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
setUp(() async {
await d.file('test.dart', '''
import 'package:test/test.dart';
diff --git a/pkgs/test/test/runner/tear_down_all_test.dart b/pkgs/test/test/runner/tear_down_all_test.dart
index 67b4808..1304ccc 100644
--- a/pkgs/test/test/runner/tear_down_all_test.dart
+++ b/pkgs/test/test/runner/tear_down_all_test.dart
@@ -11,6 +11,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('an error causes the run to fail', () async {
await d.file('test.dart', r'''
import 'package:test/test.dart';
diff --git a/pkgs/test/test/runner/test_chain_test.dart b/pkgs/test/test/runner/test_chain_test.dart
index 5b83acb..e7c1da0 100644
--- a/pkgs/test/test/runner/test_chain_test.dart
+++ b/pkgs/test/test/runner/test_chain_test.dart
@@ -12,6 +12,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
setUp(() async {
await d.file('test.dart', '''
import 'dart:async';
diff --git a/pkgs/test/test/runner/test_on_test.dart b/pkgs/test/test/runner/test_on_test.dart
index 95a39bd..47c510f 100644
--- a/pkgs/test/test/runner/test_on_test.dart
+++ b/pkgs/test/test/runner/test_on_test.dart
@@ -20,6 +20,7 @@
late PackageConfig currentPackageConfig;
setUpAll(() async {
+ await precompileTestExecutable();
currentPackageConfig =
await loadPackageConfigUri((await Isolate.packageConfig)!);
});
diff --git a/pkgs/test/test/runner/timeout_test.dart b/pkgs/test/test/runner/timeout_test.dart
index a9756da..598b95d 100644
--- a/pkgs/test/test/runner/timeout_test.dart
+++ b/pkgs/test/test/runner/timeout_test.dart
@@ -11,6 +11,8 @@
import '../io.dart';
void main() {
+ setUpAll(precompileTestExecutable);
+
test('respects top-level @Timeout declarations', () async {
await d.file('test.dart', '''
@Timeout(const Duration(seconds: 0))
