[test] Add compiler `dart test -c cli` to run hooks (#2660)
Bug:
* https://github.com/dart-lang/sdk/issues/63372
This PR uses `dart build cli` under the `-c cli` flag.
Implementation:
* The wrapper Dart script around the test that package test creates is the new entry-point and **must** be within the package root to run the correct hooks.
* So, this PR updates the temp dir to be in `.dart_tool/test/temp/<...>` rather than in the system temp.
* Gates the implementation on the next dev release after https://dart-review.googlesource.com/c/sdk/+/506242.
Testing:
* The integration tests run with the `cli` compiler
* The integration tests must run on a valid package with a valid pubspec and package_config.json (otherwise the hooks-runner cannot determine which hooks to run).
* The integration tests do _not_ actually run a hook and include c code. `dart build cli` bundles are already properly tested in the dartdev tests for `dart build cli` in the SDK. If you feel we should add an integration test running native code here, I'm happy to add one.
* Skips the tests before the next dev release.
* **TODO**: Rerun tests on Tuesday after new dev release has come out.
diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md
index c148eac..438b506 100644
--- a/pkgs/test/CHANGELOG.md
+++ b/pkgs/test/CHANGELOG.md
@@ -1,5 +1,7 @@
## 1.31.2-wip
+* Add support for running tests as native CLI bundles (vm platform only).
+ * You can run tests this way with `--compiler cli`.
* **Impacts Configuration** Support using the OS platform selector to configure
browser tests.
Previously tests loaded for the browser would have an operating system of
diff --git a/pkgs/test/lib/src/runner/browser/compilers/precompiled.dart b/pkgs/test/lib/src/runner/browser/compilers/precompiled.dart
index f5d2d78..77fd36d 100644
--- a/pkgs/test/lib/src/runner/browser/compilers/precompiled.dart
+++ b/pkgs/test/lib/src/runner/browser/compilers/precompiled.dart
@@ -112,6 +112,7 @@
faviconPath,
),
Compiler.exe ||
+ Compiler.cli ||
Compiler.kernel ||
Compiler.source => throw UnsupportedError(
'The browser platform does not support $compiler',
diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml
index 9cbda09..8c8f47c 100644
--- a/pkgs/test/pubspec.yaml
+++ b/pkgs/test/pubspec.yaml
@@ -46,6 +46,7 @@
dev_dependencies:
fake_async: ^1.0.0
glob: ^2.0.0
+ pub_semver: ^2.0.0
test_descriptor: ^2.0.0
test_process: ^2.0.0
diff --git a/pkgs/test/test/io.dart b/pkgs/test/test/io.dart
index ef7eb96..d1780be 100644
--- a/pkgs/test/test/io.dart
+++ b/pkgs/test/test/io.dart
@@ -7,6 +7,7 @@
import 'dart:isolate';
import 'package:path/path.dart' as p;
+import 'package:pub_semver/pub_semver.dart';
import 'package:test/test.dart';
import 'package:test_descriptor/test_descriptor.dart' as d;
import 'package:test_process/test_process.dart';
@@ -25,6 +26,11 @@
/// The root directory of the Dart SDK.
final String sdkDir = p.dirname(p.dirname(Platform.resolvedExecutable));
+final bool supportsCliCompiler = () {
+ var current = Version.parse(Platform.version.split(' ').first);
+ return current > Version.parse('3.13.0-159.0.dev');
+}();
+
/// The platform-specific message emitted when a nonexistent file is loaded.
final String noSuchFileMessage = Platform.isWindows
? 'The system cannot find the file specified.'
diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart
index 6d90a2d..f6ec7da 100644
--- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart
+++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart
@@ -48,6 +48,8 @@
} else if (runtime == Runtime.vmTsan &&
!File('$sdkDir/bin/dartaotruntime_tsan').existsSync()) {
skipReason = 'SDK too old';
+ } else if (compiler == Compiler.cli && !supportsCliCompiler) {
+ skipReason = 'SDK too old';
}
group(
'--runtime ${runtime.identifier} --compiler ${compiler.identifier}',
diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart
index 8a8594f..5647ef9 100644
--- a/pkgs/test/test/runner/runner_test.dart
+++ b/pkgs/test/test/runner/runner_test.dart
@@ -75,7 +75,7 @@
$_runtimes.
Each platform supports the following compilers:
$_runtimeCompilers
--c, --compiler The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, exe, kernel, source].
+-c, --compiler The compiler(s) to use to run tests, supported compilers are [dart2js, dart2wasm, exe, cli, kernel, source].
Each platform has a default compiler but may support other compilers.
You can target a compiler to a specific platform using arguments of the following form [<platform-selector>:]<compiler>.
If a platform is specified but no given compiler is supported for that platform, then it will use its default compiler.
@@ -137,7 +137,7 @@
', edge, node]';
final _runtimeCompilers = [
- '[vm]: kernel (default), source, exe',
+ '[vm]: kernel (default), source, exe, cli',
'[vm-asan]: exe (default)',
'[vm-msan]: exe (default)',
'[vm-tsan]: exe (default)',
diff --git a/pkgs/test/test/runner/subprocess_crash_test.dart b/pkgs/test/test/runner/subprocess_crash_test.dart
index e98f634..59bea2f 100644
--- a/pkgs/test/test/runner/subprocess_crash_test.dart
+++ b/pkgs/test/test/runner/subprocess_crash_test.dart
@@ -13,27 +13,35 @@
void main() {
setUpAll(precompileTestExecutable);
- test('gracefully handles an early test suite exit', () async {
- await d.file('test.dart', '''
- import 'dart:io';
+ for (var compiler in ['exe', 'cli']) {
+ test(
+ 'gracefully handles an early test suite exit with the $compiler compiler',
+ () async {
+ await d.file('test.dart', '''
+ import 'dart:io';
- import 'package:test/test.dart';
+ import 'package:test/test.dart';
- void main() {
- test('runs', () {});
- test('exits', () {
- exit(0);
- });
- }''').create();
+ void main() {
+ test('runs', () {});
+ test('exits', () {
+ exit(0);
+ });
+ }''').create();
- var test = await runTest(['--compiler', 'exe', 'test.dart']);
- expect(
- test.stdout,
- containsInOrder([
- '+1: [VM, Exe] exits - did not complete [E]',
- '+1: Some tests failed.',
- ]),
+ var test = await runTest(['--compiler', compiler, 'test.dart']);
+ expect(
+ test.stdout,
+ containsInOrder([
+ '+1: [VM, ${compiler == 'exe' ? 'Exe' : 'Cli'}] exits - did not complete [E]',
+ '+1: Some tests failed.',
+ ]),
+ );
+ await test.shouldExit(1);
+ },
+ skip: compiler == 'cli' && !supportsCliCompiler
+ ? 'Dart version does not support build cli'
+ : null,
);
- await test.shouldExit(1);
- });
+ }
}
diff --git a/pkgs/test_api/CHANGELOG.md b/pkgs/test_api/CHANGELOG.md
index 3f36614..cfc807c 100644
--- a/pkgs/test_api/CHANGELOG.md
+++ b/pkgs/test_api/CHANGELOG.md
@@ -1,5 +1,6 @@
## 0.7.13-wip
+* Add `Compiler.cli` (the native CLI compiler).
* Expose several more backend APIs for use in `flutter_test`.
* Allow using browser platforms with a specified OS.
diff --git a/pkgs/test_api/lib/src/backend/compiler.dart b/pkgs/test_api/lib/src/backend/compiler.dart
index dc41e92..5df6bf6 100644
--- a/pkgs/test_api/lib/src/backend/compiler.dart
+++ b/pkgs/test_api/lib/src/backend/compiler.dart
@@ -10,9 +10,14 @@
/// Experimental Dart to Wasm compiler.
dart2wasm('Dart2Wasm', 'dart2wasm'),
- /// Compiles dart code to a native executable.
+ /// Compiles dart code to a native executable. This compiler does not
+ /// support code assets from build and link hooks.
exe('Exe', 'exe'),
+ /// Compiles dart code to a native CLI bundle. This compiler supports
+ /// code assets from build and link hooks.
+ cli('Cli', 'cli'),
+
/// The standard compiler for vm tests, compiles tests to kernel before
/// running them on the VM.
kernel('Kernel', 'kernel'),
@@ -25,6 +30,7 @@
Compiler.dart2js,
Compiler.dart2wasm,
Compiler.exe,
+ Compiler.cli,
Compiler.kernel,
Compiler.source,
];
diff --git a/pkgs/test_api/lib/src/backend/runtime.dart b/pkgs/test_api/lib/src/backend/runtime.dart
index 0ce9cd7..3e1e4cd 100644
--- a/pkgs/test_api/lib/src/backend/runtime.dart
+++ b/pkgs/test_api/lib/src/backend/runtime.dart
@@ -14,6 +14,7 @@
Compiler.kernel,
Compiler.source,
Compiler.exe,
+ Compiler.cli,
], isDartVM: true);
static const Runtime vmAsan = Runtime(
'VM with Address Sanitizer',
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index 6907873..d77a622 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -1,5 +1,6 @@
## 0.6.19-wip
+* Add support for `-c cli` (the native CLI compiler) to the vm platform.
* Support using the OS platform selector to configure browser tests.
* Use a DevTools URL instead of a defunct observatory URL.
diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart
index 8916e77..1c1f596 100644
--- a/pkgs/test_core/lib/src/runner/vm/platform.dart
+++ b/pkgs/test_core/lib/src/runner/vm/platform.dart
@@ -57,7 +57,8 @@
MultiChannel outerChannel;
var cleanupCallbacks = <void Function()>[];
Isolate? isolate;
- if (platform.compiler == Compiler.exe) {
+ if (platform.compiler == Compiler.exe ||
+ platform.compiler == Compiler.cli) {
var serverSocket = await ServerSocket.bind('localhost', 0);
Process process;
try {
@@ -131,10 +132,11 @@
String? isolateID;
Uri? serverUri;
if (_config.debug) {
- if (platform.compiler == Compiler.exe) {
+ if (platform.compiler == Compiler.exe ||
+ platform.compiler == Compiler.cli) {
throw UnsupportedError(
- 'Unable to debug tests compiled to `exe` (tried to debug $path with '
- 'the `exe` compiler).',
+ 'Unable to debug tests compiled to `${platform.compiler.identifier}` '
+ '(tried to debug $path with the `${platform.compiler.identifier}` compiler).',
);
}
var info = await Service.controlWebServer(
@@ -239,13 +241,79 @@
);
}
- var sharedLibrary = await _compileToNative(platform, path, suiteMetadata);
- return await Process.start(
- _aotRuntimeFor(platform),
- [sharedLibrary, socket.address.host, socket.port.toString()],
- environment: _environmentFor(platform),
- mode: ProcessStartMode.inheritStdio,
+ switch (platform.compiler) {
+ case Compiler.cli:
+ var executable = await _compileToCli(platform, path, suiteMetadata);
+ return await Process.start(executable, [
+ socket.address.host,
+ socket.port.toString(),
+ ], mode: ProcessStartMode.inheritStdio);
+ case Compiler.exe:
+ var sharedLibrary = await _compileToNative(
+ platform,
+ path,
+ suiteMetadata,
+ );
+ return await Process.start(
+ _aotRuntimeFor(platform),
+ [sharedLibrary, socket.address.host, socket.port.toString()],
+ environment: _environmentFor(platform),
+ mode: ProcessStartMode.inheritStdio,
+ );
+ default:
+ throw StateError(
+ 'Unsupported compiler ${platform.compiler} for spawning an executable',
+ );
+ }
+ }
+
+ /// Compiles [path] to a native CLI bundle using `dart build cli`.
+ Future<String> _compileToCli(
+ SuitePlatform platform,
+ String path,
+ Metadata suiteMetadata,
+ ) async {
+ var bootstrapPath = await _bootstrapNativeTestFile(
+ path,
+ suiteMetadata.languageVersionComment ??
+ await rootPackageLanguageVersionComment,
);
+
+ var outputDir = p.join(_tempDir.path, 'cli_build');
+ var processResult = await Process.run(Platform.resolvedExecutable, [
+ 'build',
+ 'cli',
+ '--target',
+ bootstrapPath,
+ '--output',
+ outputDir,
+ '--packages',
+ (await packageConfigUri).toFilePath(),
+ '--root-package',
+ (await currentPackage).name,
+ if (platform.runtime == Runtime.vmAsan) '--target-sanitizer=asan',
+ if (platform.runtime == Runtime.vmMsan) '--target-sanitizer=msan',
+ if (platform.runtime == Runtime.vmTsan) '--target-sanitizer=tsan',
+ ]);
+ if (processResult.exitCode != 0) {
+ throw LoadException(path, '''
+exitCode: ${processResult.exitCode}
+stdout: ${processResult.stdout}
+stderr: ${processResult.stderr}''');
+ }
+ var executablePath = p.join(
+ outputDir,
+ 'bundle',
+ 'bin',
+ p.basenameWithoutExtension(bootstrapPath),
+ );
+ if (!await File(executablePath).exists()) {
+ throw LoadException(
+ path,
+ 'Compiled executable not found at $executablePath',
+ );
+ }
+ return executablePath;
}
/// Compiles [path] to a native shared library using
