[test] Support CLI compiler for sanitizer runtimes (#2708)
diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md index 8df1963..d831460 100644 --- a/pkgs/test/CHANGELOG.md +++ b/pkgs/test/CHANGELOG.md
@@ -1,5 +1,7 @@ ## 1.32.0-wip +* Support `--compiler cli` with the `vm-asan`, `vm-msan`, and `vm-tsan` + runtimes. * Add support for `DART_TEST_REPORTER` environment variable in test runner and when tests are run directly on platforms which support `dart:io`. The environment variable takes precedence over configuration in `dart_test.yaml`
diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml index 3a7bba6..09bc1a0 100644 --- a/pkgs/test/pubspec.yaml +++ b/pkgs/test/pubspec.yaml
@@ -35,7 +35,7 @@ stream_channel: ^2.1.0 # Use an exact version until the test_api and test_core package are stable. - test_api: 0.7.13 + test_api: 0.7.14-wip test_core: 0.6.20-wip typed_data: ^1.3.0
diff --git a/pkgs/test/test/io.dart b/pkgs/test/test/io.dart index e6ec47b..b99250a 100644 --- a/pkgs/test/test/io.dart +++ b/pkgs/test/test/io.dart
@@ -31,6 +31,11 @@ return current > Version.parse('3.13.0-159.0.dev'); }(); +final bool supportsCliCompilerSanitizers = () { + var current = Version.parse(Platform.version.split(' ').first); + return current > Version.parse('3.14.0-62.0.dev'); +}(); + /// The platform-specific message emitted when a nonexistent file is loaded. final String noSuchFileMessage = Platform.isWindows ? 'The system cannot find the file specified'
diff --git a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart index f6ec7da..035ba9e 100644 --- a/pkgs/test/test/runner/compiler_runtime_matrix_test.dart +++ b/pkgs/test/test/runner/compiler_runtime_matrix_test.dart
@@ -21,6 +21,18 @@ for (var runtime in Runtime.builtIn) { for (var compiler in runtime.supportedCompilers) { + final sanitizerSuffix = switch (runtime) { + Runtime.vmAsan => 'asan', + Runtime.vmMsan => 'msan', + Runtime.vmTsan => 'tsan', + _ => null, + }; + final sanitizerEnvironment = switch (runtime) { + Runtime.vmAsan => 'ASAN_OPTIONS', + Runtime.vmMsan => 'MSAN_OPTIONS', + Runtime.vmTsan => 'TSAN_OPTIONS', + _ => null, + }; // Ignore the platforms we can't run on this OS. if ((runtime == Runtime.edge && !Platform.isWindows) || (runtime == Runtime.safari && !Platform.isMacOS) || @@ -39,17 +51,19 @@ skipReason = 'https://github.com/dart-lang/test/issues/1942'; } else if (runtime == Runtime.firefox && Platform.isMacOS) { skipReason = 'https://github.com/dart-lang/test/pull/2276'; - } else if (runtime == Runtime.vmAsan && - !File('$sdkDir/bin/dartaotruntime_asan').existsSync()) { - skipReason = 'SDK too old'; - } else if (runtime == Runtime.vmMsan && - !File('$sdkDir/bin/dartaotruntime_msan').existsSync()) { - skipReason = 'SDK too old'; - } else if (runtime == Runtime.vmTsan && - !File('$sdkDir/bin/dartaotruntime_tsan').existsSync()) { + } else if (compiler == Compiler.cli && + sanitizerSuffix != null && + !supportsCliCompilerSanitizers) { skipReason = 'SDK too old'; } else if (compiler == Compiler.cli && !supportsCliCompiler) { skipReason = 'SDK too old'; + } else if (sanitizerSuffix != null && + !File( + '$sdkDir/bin/' + '${compiler == Compiler.cli ? 'dartcliruntime' : 'dartaotruntime'}' + '_$sanitizerSuffix', + ).existsSync()) { + skipReason = 'SDK too old'; } group( '--runtime ${runtime.identifier} --compiler ${compiler.identifier}', @@ -149,6 +163,43 @@ : null, ); } + + if (sanitizerEnvironment != null && compiler == Compiler.cli) { + test( + 'sets sanitizer environment defaults', + () async { + await d + .file( + 'test.dart', + _sanitizerEnvironmentTest( + sanitizerEnvironment, + 'halt_on_error=1:exitcode=6:symbolize=1', + ), + ) + .create(); + var test = await runTest(testArgs); + await test.shouldExit(0); + }, + skip: Platform.environment.containsKey(sanitizerEnvironment) + ? '$sanitizerEnvironment is set by the test environment' + : null, + ); + + test('preserves sanitizer environment options', () async { + const options = 'halt_on_error=1:exitcode=7:symbolize=1'; + await d + .file( + 'test.dart', + _sanitizerEnvironmentTest(sanitizerEnvironment, options), + ) + .create(); + var test = await runTest( + testArgs, + environment: {sanitizerEnvironment: options}, + ); + await test.shouldExit(0); + }); + } }, ); } @@ -199,3 +250,16 @@ stderr.writeln('world'); test('success', () {}); }'''; + +String _sanitizerEnvironmentTest(String name, String value) => + ''' +import 'dart:io'; + +import 'package:test/test.dart'; + +void main() { + test('sanitizer environment', () { + expect(Platform.environment['$name'], '$value'); + }); +} +''';
diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart index 3a945ba..b919e9e 100644 --- a/pkgs/test/test/runner/runner_test.dart +++ b/pkgs/test/test/runner/runner_test.dart
@@ -138,9 +138,9 @@ final _runtimeCompilers = [ '[vm]: kernel (default), source, exe, cli', - '[vm-asan]: exe (default)', - '[vm-msan]: exe (default)', - '[vm-tsan]: exe (default)', + '[vm-asan]: exe (default), cli', + '[vm-msan]: exe (default), cli', + '[vm-tsan]: exe (default), cli', '[chrome]: dart2js (default), dart2wasm', '[firefox]: dart2js (default), dart2wasm', if (Platform.isMacOS) '[safari]: dart2js (default)',
diff --git a/pkgs/test_api/CHANGELOG.md b/pkgs/test_api/CHANGELOG.md index 703f098..169c415 100644 --- a/pkgs/test_api/CHANGELOG.md +++ b/pkgs/test_api/CHANGELOG.md
@@ -1,3 +1,8 @@ +## 0.7.14-wip + +* Allow the `vmAsan`, `vmMsan`, and `vmTsan` runtimes to use the native CLI + compiler. + ## 0.7.13 * Add `Compiler.cli` (the native CLI compiler).
diff --git a/pkgs/test_api/lib/src/backend/runtime.dart b/pkgs/test_api/lib/src/backend/runtime.dart index 3e1e4cd..d25af4a 100644 --- a/pkgs/test_api/lib/src/backend/runtime.dart +++ b/pkgs/test_api/lib/src/backend/runtime.dart
@@ -20,21 +20,21 @@ 'VM with Address Sanitizer', 'vm-asan', Compiler.exe, - [Compiler.exe], + [Compiler.exe, Compiler.cli], isDartVM: true, ); static const Runtime vmMsan = Runtime( 'VM with Memory Sanitizer', 'vm-msan', Compiler.exe, - [Compiler.exe], + [Compiler.exe, Compiler.cli], isDartVM: true, ); static const Runtime vmTsan = Runtime( 'VM with Thread Sanitizer', 'vm-tsan', Compiler.exe, - [Compiler.exe], + [Compiler.exe, Compiler.cli], isDartVM: true, );
diff --git a/pkgs/test_api/pubspec.yaml b/pkgs/test_api/pubspec.yaml index 09cd261..568c808 100644 --- a/pkgs/test_api/pubspec.yaml +++ b/pkgs/test_api/pubspec.yaml
@@ -1,5 +1,5 @@ name: test_api -version: 0.7.13 +version: 0.7.14-wip description: >- The user facing API for structuring Dart tests and checking expectations. repository: https://github.com/dart-lang/test/tree/master/pkgs/test_api
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md index 5306a05..2926c0b 100644 --- a/pkgs/test_core/CHANGELOG.md +++ b/pkgs/test_core/CHANGELOG.md
@@ -1,5 +1,7 @@ ## 0.6.20-wip +* Pass sanitizer environment defaults when running tests compiled as native CLI + bundles. * Add support for `DART_TEST_REPORTER` environment variable. * Suppress dart2js compiler output for successful compiles. * Include passing test messages within the `Passing tests` group in
diff --git a/pkgs/test_core/lib/src/runner/vm/platform.dart b/pkgs/test_core/lib/src/runner/vm/platform.dart index 28af166..231cf6f 100644 --- a/pkgs/test_core/lib/src/runner/vm/platform.dart +++ b/pkgs/test_core/lib/src/runner/vm/platform.dart
@@ -245,10 +245,12 @@ switch (platform.compiler) { case Compiler.cli: var executable = await _compileToCli(platform, path, suiteMetadata); - return await Process.start(executable, [ - socket.address.host, - socket.port.toString(), - ], mode: ProcessStartMode.inheritStdio); + return await Process.start( + executable, + [socket.address.host, socket.port.toString()], + environment: _environmentFor(platform), + mode: ProcessStartMode.inheritStdio, + ); case Compiler.exe: var sharedLibrary = await _compileToNative( platform,
diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml index e5f7125..ddebc64 100644 --- a/pkgs/test_core/pubspec.yaml +++ b/pkgs/test_core/pubspec.yaml
@@ -28,7 +28,7 @@ stack_trace: ^1.10.0 stream_channel: ^2.1.0 # Use an exact version until the test_api package is stable. - test_api: 0.7.13 + test_api: 0.7.14-wip vm_service: '>=6.0.0 <16.0.0' yaml: ^3.0.0