[testing] Use migrated suites by default on non-legacy configurations
Make test.py pick migrated suites by default when the `NnbdMode` is not `NnbdMode.legacy`.
* Remove non-existing benchmark_smoke suite.
* Remove broken analyzer_library suite from default suites.
* Deprecated observatory_ui is not added to the default migrated suites.
* Remove remaining references to samples-dev.
* Remove unnecessary suite specifications from the test_matrix.json.
* Remove broken references to observatory_ui from the test_matrix.json.
* Remove defunct observatory_ui hack from test.py.
Fixes: b/268474066
Change-Id: I6c3635bcb396021ca86498d60c9efc77ab39589e
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/281701
Reviewed-by: William Hesse <whesse@google.com>
Commit-Queue: Alexander Thomas <athom@google.com>
diff --git a/pkg/test_runner/lib/src/options.dart b/pkg/test_runner/lib/src/options.dart
index 30bd545..dddf4d1 100644
--- a/pkg/test_runner/lib/src/options.dart
+++ b/pkg/test_runner/lib/src/options.dart
@@ -15,20 +15,30 @@
import 'test_configurations.dart';
import 'utils.dart';
-const _defaultTestSelectors = [
- 'samples',
- 'standalone_2',
+const _legacyTestSelectors = [
'corelib_2',
+ 'ffi_2',
'language_2',
- 'vm',
- 'benchmark_smoke',
- 'utils',
'lib_2',
- 'analyze_library',
- 'service_2',
'kernel',
'observatory_ui_2',
- 'ffi_2'
+ 'service_2',
+ 'standalone_2',
+ 'utils',
+ 'vm',
+];
+
+const _defaultTestSelectors = [
+ 'corelib',
+ 'ffi',
+ 'kernel',
+ 'language',
+ 'lib',
+ 'samples',
+ 'service',
+ 'standalone',
+ 'utils',
+ 'vm',
];
extension _IntOption on ArgParser {
@@ -540,7 +550,7 @@
options['test-list-contents'] = LineSplitter.split(tests).toList();
}
- return _createConfigurations(options);
+ return _expandConfigurations(options);
}
/// Given a set of parsed option values, returns the list of command line
@@ -583,48 +593,9 @@
return arguments;
}
- List<TestConfiguration> _createConfigurations(
- Map<String, dynamic> configuration) {
- var selectors = _expandSelectors(configuration);
-
- // Put observatory_ui in a configuration with its own packages override.
- // Only one value in the configuration map is mutable:
- if (selectors.containsKey('observatory_ui')) {
- if (selectors.length == 1) {
- configuration['packages'] = Repository.uri
- .resolve('.dart_tool/package_config.json')
- .toFilePath();
- } else {
- // Make a new configuration whose selectors map only contains
- // observatory_ui, and remove observatory_ui from the original
- // selectors. The only mutable value in the map is the selectors, so a
- // shallow copy is safe.
- var observatoryConfiguration = Map<String, dynamic>.from(configuration);
- var observatorySelectors = {
- 'observatory_ui': selectors['observatory_ui']
- };
- selectors.remove('observatory_ui');
-
- // Set the packages flag.
- observatoryConfiguration['packages'] = Repository.uri
- .resolve('.dart_tool/package_config.json')
- .toFilePath();
-
- return [
- ..._expandConfigurations(configuration, selectors),
- ..._expandConfigurations(
- observatoryConfiguration, observatorySelectors)
- ];
- }
- }
-
- return _expandConfigurations(configuration, selectors);
- }
-
/// Recursively expands a configuration with multiple values per key into a
/// list of configurations with exactly one value per key.
- List<TestConfiguration> _expandConfigurations(
- Map<String, dynamic> data, Map<String, RegExp?> selectors) {
+ List<TestConfiguration> _expandConfigurations(Map<String, dynamic> data) {
var result = <TestConfiguration>[];
// Handles a string option containing a space-separated list of words.
@@ -687,7 +658,7 @@
var configuration = TestConfiguration(
configuration: innerConfiguration,
progress: progress,
- selectors: selectors,
+ selectors: _expandSelectors(data, innerConfiguration.nnbdMode),
build: data["build"] as bool,
testList: data["test-list-contents"] as List<String>?,
repeat: int.parse(data["repeat"] as String),
@@ -869,7 +840,8 @@
/// expression to be used on the full path of a test file in that test suite.
///
/// If no selectors are explicitly given, uses the default suite patterns.
- Map<String, RegExp> _expandSelectors(Map<String, dynamic> configuration) {
+ Map<String, RegExp> _expandSelectors(
+ Map<String, dynamic> configuration, NnbdMode nnbdMode) {
var selectors = configuration['selectors'] as List<String>?;
if (selectors == null || selectors.isEmpty) {
@@ -882,7 +854,11 @@
.toSet()
.toList();
} else {
- selectors = _defaultTestSelectors.toList();
+ if (nnbdMode == NnbdMode.legacy) {
+ selectors = _legacyTestSelectors.toList();
+ } else {
+ selectors = _defaultTestSelectors.toList();
+ }
}
var excludeSuites = configuration['exclude-suite'] != null
diff --git a/pkg/test_runner/lib/src/test_configurations.dart b/pkg/test_runner/lib/src/test_configurations.dart
index 5de2eb9..002549f 100644
--- a/pkg/test_runner/lib/src/test_configurations.dart
+++ b/pkg/test_runner/lib/src/test_configurations.dart
@@ -35,7 +35,6 @@
Path('runtime/observatory_2/tests/service_2'),
Path('runtime/tests/vm'),
Path('samples'),
- Path('samples-dev'),
Path('tests/corelib'),
Path('tests/corelib_2'),
Path('tests/dartdevc'),
diff --git a/pkg/test_runner/test/options_test.dart b/pkg/test_runner/test/options_test.dart
index ba771e7..39694ac 100644
--- a/pkg/test_runner/test/options_test.dart
+++ b/pkg/test_runner/test/options_test.dart
@@ -10,6 +10,7 @@
testDefaults();
testOptions();
testValidation();
+ testSelectors();
}
void testDefaults() {
@@ -99,6 +100,48 @@
'The named configuration "invalid-vm-android-simarm" is invalid.');
}
+void testSelectors() {
+ // Legacy suites.
+ for (var arguments in [
+ <String>[],
+ ['-nvm-legacy']
+ ]) {
+ var configuration = parseConfiguration(arguments);
+ Expect.setEquals({
+ 'standalone_2',
+ 'corelib_2',
+ 'language_2',
+ 'vm',
+ 'utils',
+ 'lib_2',
+ 'service_2',
+ 'kernel',
+ 'observatory_ui_2',
+ 'ffi_2',
+ }, configuration.selectors.keys, "suites for $arguments");
+ }
+
+ // Default null safe suites.
+ for (var arguments in [
+ ['--nnbd=strong'],
+ ['-nvm-strong']
+ ]) {
+ var configuration = parseConfiguration(arguments);
+ Expect.setEquals({
+ 'samples',
+ 'standalone',
+ 'corelib',
+ 'language',
+ 'vm',
+ 'utils',
+ 'lib',
+ 'service',
+ 'kernel',
+ 'ffi',
+ }, configuration.selectors.keys, "suites for $arguments");
+ }
+}
+
TestConfiguration parseConfiguration(List<String> arguments) {
var configurations = parseConfigurations(arguments);
Expect.equals(1, configurations.length);
diff --git a/pkg/test_runner/test/test_matrix.json b/pkg/test_runner/test/test_matrix.json
index 3e88049..86ee96c 100644
--- a/pkg/test_runner/test/test_matrix.json
+++ b/pkg/test_runner/test/test_matrix.json
@@ -7,6 +7,8 @@
"options": {
"compiler": "dartkp"
}
- }
+ },
+ "vm-legacy": {},
+ "vm-strong": {}
}
}
\ No newline at end of file
diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json
index 4e39648..956accc 100644
--- a/tools/bots/test_matrix.json
+++ b/tools/bots/test_matrix.json
@@ -19,7 +19,6 @@
"out/ReleaseX64/gen/utils/dartdevc/",
"pkg/",
"runtime/tests/",
- "samples-dev/",
"sdk/",
"tests/.dart_tool/package_config.json",
"tests/angular/",
@@ -65,7 +64,6 @@
"out/ReleaseX64/gen/utils/dartdevc/",
"pkg/",
"runtime/tests/",
- "samples-dev/",
"samples/",
"sdk/",
"tests/.dart_tool/package_config.json",
@@ -106,7 +104,6 @@
"out/ReleaseX64/dart2js_platform_unsound.dill",
"pkg/",
"runtime/tests/",
- "samples-dev/",
"sdk/",
"tests/.dart_tool/package_config.json",
"tests/angular/",
@@ -151,7 +148,6 @@
"out/ReleaseX64/dart2js_platform_unsound.dill",
"pkg/",
"runtime/tests/",
- "samples-dev/",
"samples/",
"sdk/",
"tests/.dart_tool/package_config.json",
@@ -190,7 +186,6 @@
"out/ReleaseX64/wasm/",
"pkg/",
"runtime/tests/",
- "samples-dev/",
"samples/",
"sdk/",
"tests/.dart_tool/package_config.json",
@@ -220,7 +215,6 @@
"out/ReleaseX64/",
"pkg/",
"runtime/tests/",
- "samples-dev/",
"samples/",
"sdk/",
"tests/.dart_tool/package_config.json",
@@ -284,7 +278,6 @@
"xcodebuild/",
"pkg/",
"samples/",
- "samples-dev/",
"tools/",
"third_party/android_tools/sdk/platform-tools/adb",
"third_party/android_tools/ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin/aarch64-linux-android-strip",
@@ -1449,17 +1442,7 @@
{
"name": "vm nnbd tests in strong mode",
"arguments": [
- "-ndartkp-strong-${system}-${mode}-${arch}",
- "benchmark_smoke",
- "corelib",
- "ffi",
- "language",
- "lib",
- "samples",
- "service",
- "standalone",
- "utils",
- "vm"
+ "-ndartkp-strong-${system}-${mode}-${arch}"
]
}
]
@@ -1485,17 +1468,7 @@
{
"name": "vm nnbd tests in strong mode",
"arguments": [
- "-ndartkp-strong-${system}-${mode}-${arch}",
- "benchmark_smoke",
- "corelib",
- "ffi",
- "language",
- "lib",
- "samples",
- "service",
- "standalone",
- "utils",
- "vm"
+ "-ndartkp-strong-${system}-${mode}-${arch}"
],
"fileset": "vm-kernel",
"shards": 6
@@ -1532,17 +1505,7 @@
{
"name": "vm nnbd tests in weak mode with asserts",
"arguments": [
- "-ndartkp-weak-asserts-${system}-${mode}-${arch}",
- "benchmark_smoke",
- "corelib",
- "ffi",
- "language",
- "lib",
- "samples",
- "service",
- "standalone",
- "utils",
- "vm"
+ "-ndartkp-weak-asserts-${system}-${mode}-${arch}"
],
"fileset": "vm-kernel",
"shards": 6
@@ -1550,17 +1513,7 @@
{
"name": "vm nnbd tests in strong mode",
"arguments": [
- "-ndartkp-strong-${system}-${mode}-${arch}",
- "benchmark_smoke",
- "corelib",
- "ffi",
- "language",
- "lib",
- "samples",
- "service",
- "standalone",
- "utils",
- "vm"
+ "-ndartkp-strong-${system}-${mode}-${arch}"
],
"fileset": "vm-kernel",
"shards": 6
@@ -1773,7 +1726,6 @@
"name": "vm tests",
"arguments": [
"-nvm-${sanitizer}-${system}-${mode}-${arch}",
- "benchmark_smoke",
"corelib",
"ffi",
"language",
@@ -2077,17 +2029,7 @@
{
"name": "vm nnbd tests in strong mode",
"arguments": [
- "-ndartk-strong-${system}-${mode}-${arch}",
- "benchmark_smoke",
- "corelib",
- "ffi",
- "language",
- "lib",
- "samples",
- "service",
- "standalone",
- "utils",
- "vm"
+ "-ndartk-strong-${system}-${mode}-${arch}"
],
"fileset": "vm-kernel",
"shards": 6
@@ -2126,17 +2068,7 @@
{
"name": "vm nnbd tests in weak mode with asserts",
"arguments": [
- "-ndartk-weak-asserts-${system}-${mode}-${arch}",
- "benchmark_smoke",
- "corelib",
- "ffi",
- "language",
- "lib",
- "samples",
- "service",
- "standalone",
- "utils",
- "vm"
+ "-ndartk-weak-asserts-${system}-${mode}-${arch}"
],
"fileset": "vm-kernel",
"shards": 6
@@ -2153,17 +2085,7 @@
{
"name": "vm nnbd tests in strong mode",
"arguments": [
- "-ndartk-strong-${system}-${mode}-${arch}",
- "benchmark_smoke",
- "corelib",
- "ffi",
- "language",
- "lib",
- "samples",
- "service",
- "standalone",
- "utils",
- "vm"
+ "-ndartk-strong-${system}-${mode}-${arch}"
],
"fileset": "vm-kernel",
"shards": 6
@@ -2334,17 +2256,7 @@
{
"name": "vm tests",
"arguments": [
- "-nvm-appjit-${system}-${mode}-${arch}",
- "benchmark_smoke",
- "corelib",
- "ffi",
- "language",
- "lib",
- "samples",
- "service",
- "standalone",
- "utils",
- "vm"
+ "-nvm-appjit-${system}-${mode}-${arch}"
],
"shards": 6,
"fileset": "vm-kernel"
@@ -2712,7 +2624,6 @@
"name": "vm tests",
"arguments": [
"-nvm-eager-optimization-linux-release-${arch}",
- "benchmark_smoke",
"corelib",
"ffi",
"language",
@@ -2747,7 +2658,6 @@
"name": "vm tests",
"arguments": [
"-nvm-reload-${system}-${mode}-${arch}",
- "benchmark_smoke",
"corelib",
"ffi",
"language",
@@ -2782,7 +2692,6 @@
"name": "vm tests",
"arguments": [
"-nvm-reload-rollback-linux-${mode}-${arch}",
- "benchmark_smoke",
"corelib",
"ffi",
"language",
@@ -2928,8 +2837,7 @@
{
"name": "dart2js tests",
"arguments": [
- "-ndart2js-hostasserts-linux-ia32-d8",
- "--exclude-suite=observatory_ui"
+ "-ndart2js-hostasserts-linux-ia32-d8"
],
"shards": 6,
"fileset": "web_platform_hostasserts"
@@ -2983,8 +2891,8 @@
"name": "dart2js tests",
"arguments": [
"-ndart2js-${system}-${runtime}",
- "--reset-browser-configuration",
- "--exclude-suite=observatory_ui"
+ "--reset-browser-configuration"
+
],
"shards": 6,
"fileset": "web_platform"
@@ -3035,8 +2943,8 @@
{
"name": "dart2js tests",
"arguments": [
- "-ndart2js-minified-linux-d8",
- "--exclude-suite=observatory_ui"
+ "-ndart2js-minified-linux-d8"
+
],
"shards": 6,
"fileset": "web_platform"
@@ -3058,8 +2966,8 @@
{
"name": "dart2js production tests",
"arguments": [
- "-ndart2js-production-linux-d8",
- "--exclude-suite=observatory_ui"
+ "-ndart2js-production-linux-d8"
+
],
"shards": 6,
"fileset": "web_platform"
@@ -3108,8 +3016,8 @@
"name": "dart2js tests",
"arguments": [
"-ndart2js-minified-csp-linux-chrome",
- "--reset-browser-configuration",
- "--exclude-suite=observatory_ui"
+ "--reset-browser-configuration"
+
],
"shards": 4,
"fileset": "web_platform"
@@ -4034,7 +3942,6 @@
"name": "vm tests",
"arguments": [
"-ndartkp-strong-${system}-${mode}-simarm-crossword",
- "benchmark_smoke",
"corelib",
"ffi",
"language",
