Allow disabling test randomization for certain tests (#1551)
Fixes #1539
Adds an allow_test_randomization option that can be configured for certain tags, in the dart_test.yml file.
Also restructured the Configuration and SuiteConfiguration classes to make future changes less error prone, by making their parameters required. I also added some specialized constructors for the few cases where we really only want to pass a couple options.
diff --git a/pkgs/test/doc/configuration.md b/pkgs/test/doc/configuration.md
index 7a7cabb..0c78b53 100644
--- a/pkgs/test/doc/configuration.md
+++ b/pkgs/test/doc/configuration.md
@@ -32,6 +32,7 @@
* [`skip`](#skip)
* [`retry`](#retry)
* [`test_on`](#test_on)
+ * [`allow_test_randomization`](#allow_test_randomization)
* [Runner Configuration](#runner-configuration)
* [`include`](#include)
* [`paths`](#paths)
@@ -46,6 +47,7 @@
* [`run_skipped`](#run_skipped)
* [`pub_serve`](#pub_serve)
* [`reporter`](#reporter)
+ * [`file_reporters`](#file_reporters)
* [`fold_stack_frames`](#fold_stack_frames)
* [`custom_html_template_path`](#custom_html_template_path)
* [Configuring Tags](#configuring-tags)
@@ -56,13 +58,13 @@
* [`on_platform`](#on_platform)
* [`override_platforms`](#override_platforms)
* [`define_platforms`](#define_platforms)
- * [Browser/Node.js Settings](#browser-and-node-js-settings)
+ * [Browser and Node.js Settings](#browser-and-nodejs-settings)
* [`arguments`](#arguments)
* [`executable`](#executable)
* [`headless`](#headless)
* [Configuration Presets](#configuration-presets)
* [`presets`](#presets)
- * [`add_preset`](#add_preset)
+ * [`add_presets`](#add_presets)
* [Global Configuration](#global-configuration)
## Test Configuration
@@ -191,6 +193,20 @@
This field is not supported in the
[global configuration file](#global-configuration).
+### `allow_test_randomization`
+
+This can be used to disable test randomization for certain tests, regardless
+of the `--test-randomize-ordering-seed` configuration.
+
+This is typically useful when a subset of your tests are order dependent, but
+you want to run the other ones with randomized ordering.
+
+```yaml
+tags:
+ doNotRandomize:
+ allow_test_randomization: false
+```
+
## Runner Configuration
Unlike [test configuration](#test-configuration), runner configuration affects
diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml
index be4baba..509b337 100644
--- a/pkgs/test/pubspec.yaml
+++ b/pkgs/test/pubspec.yaml
@@ -34,10 +34,11 @@
yaml: ^3.0.0
# Use an exact version until the test_api and test_core package are stable.
test_api: 0.4.1
- test_core: 0.3.30
+ test_core: 0.4.0
dev_dependencies:
fake_async: ^1.0.0
+ glob: ^2.0.0
shelf_test_handler: ^2.0.0
test_descriptor: ^2.0.0
test_process: ^2.0.0
diff --git a/pkgs/test/test/runner/browser/chrome_test.dart b/pkgs/test/test/runner/browser/chrome_test.dart
index a60bdf4..b5ac51e 100644
--- a/pkgs/test/test/runner/browser/chrome_test.dart
+++ b/pkgs/test/test/runner/browser/chrome_test.dart
@@ -8,7 +8,6 @@
import 'package:test/src/runner/browser/chrome.dart';
import 'package:test/src/runner/executable_settings.dart';
import 'package:test/test.dart';
-import 'package:test_core/src/runner/configuration.dart'; // ignore: implementation_imports
import 'package:test_descriptor/test_descriptor.dart' as d;
import '../../io.dart';
@@ -27,7 +26,7 @@
''');
var webSocket = server.handleWebSocket();
- var chrome = Chrome(server.url, Configuration());
+ var chrome = Chrome(server.url, configuration());
addTearDown(() => chrome.close());
expect(await (await webSocket).stream.first, equals('loaded!'));
@@ -38,12 +37,12 @@
test("a process can be killed synchronously after it's started", () async {
var server = await CodeServer.start();
- var chrome = Chrome(server.url, Configuration());
+ var chrome = Chrome(server.url, configuration());
await chrome.close();
});
test('reports an error in onExit', () {
- var chrome = Chrome(Uri.parse('http://dart-lang.org'), Configuration(),
+ var chrome = Chrome(Uri.parse('http://dart-lang.org'), configuration(),
settings: ExecutableSettings(
linuxExecutable: '_does_not_exist',
macOSExecutable: '_does_not_exist',
diff --git a/pkgs/test/test/runner/browser/loader_test.dart b/pkgs/test/test/runner/browser/loader_test.dart
index e1d6d37..30fa79c 100644
--- a/pkgs/test/test/runner/browser/loader_test.dart
+++ b/pkgs/test/test/runner/browser/loader_test.dart
@@ -27,7 +27,7 @@
/// A configuration that loads suites on Chrome.
final _chrome =
- SuiteConfiguration(runtimes: [RuntimeSelection(Runtime.chrome.identifier)]);
+ SuiteConfiguration.runtimes([RuntimeSelection(Runtime.chrome.identifier)]);
void main() {
setUp(() async {
@@ -135,7 +135,7 @@
var suites = await _loader
.loadFile(
path,
- SuiteConfiguration(runtimes: [
+ SuiteConfiguration.runtimes([
RuntimeSelection(Runtime.vm.identifier),
RuntimeSelection(Runtime.chrome.identifier)
]))
diff --git a/pkgs/test/test/runner/configuration/configuration_test.dart b/pkgs/test/test/runner/configuration/configuration_test.dart
index 83dee46..6538b75 100644
--- a/pkgs/test/test/runner/configuration/configuration_test.dart
+++ b/pkgs/test/test/runner/configuration/configuration_test.dart
@@ -6,15 +6,16 @@
import 'package:path/path.dart' as p;
import 'package:test/test.dart';
-import 'package:test_core/src/runner/configuration.dart';
import 'package:test_core/src/runner/configuration/reporters.dart';
import 'package:test_core/src/util/io.dart';
+import '../../utils.dart';
+
void main() {
group('merge', () {
group('for most fields', () {
test('if neither is defined, preserves the default', () {
- var merged = Configuration().merge(Configuration());
+ var merged = configuration().merge(configuration());
expect(merged.help, isFalse);
expect(merged.version, isFalse);
expect(merged.pauseAfterLoad, isFalse);
@@ -32,7 +33,7 @@
});
test("if only the old configuration's is defined, uses it", () {
- var merged = Configuration(
+ var merged = configuration(
help: true,
version: true,
pauseAfterLoad: true,
@@ -46,7 +47,7 @@
shardIndex: 3,
totalShards: 10,
testRandomizeOrderingSeed: 123,
- paths: ['bar']).merge(Configuration());
+ paths: ['bar']).merge(configuration());
expect(merged.help, isTrue);
expect(merged.version, isTrue);
@@ -65,7 +66,7 @@
});
test("if only the new configuration's is defined, uses it", () {
- var merged = Configuration().merge(Configuration(
+ var merged = configuration().merge(configuration(
help: true,
version: true,
pauseAfterLoad: true,
@@ -100,7 +101,7 @@
test(
"if the two configurations conflict, uses the new configuration's "
'values', () {
- var older = Configuration(
+ var older = configuration(
help: true,
version: false,
pauseAfterLoad: true,
@@ -115,7 +116,7 @@
totalShards: 4,
testRandomizeOrderingSeed: 0,
paths: ['bar']);
- var newer = Configuration(
+ var newer = configuration(
help: false,
version: true,
pauseAfterLoad: false,
@@ -151,37 +152,37 @@
group('for chosenPresets', () {
test('if neither is defined, preserves the default', () {
- var merged = Configuration().merge(Configuration());
+ var merged = configuration().merge(configuration());
expect(merged.chosenPresets, isEmpty);
});
test("if only the old configuration's is defined, uses it", () {
- var merged = Configuration(chosenPresets: ['baz', 'bang'])
- .merge(Configuration());
+ var merged = configuration(chosenPresets: ['baz', 'bang'])
+ .merge(configuration());
expect(merged.chosenPresets, equals(['baz', 'bang']));
});
test("if only the new configuration's is defined, uses it", () {
- var merged = Configuration()
- .merge(Configuration(chosenPresets: ['baz', 'bang']));
+ var merged = configuration()
+ .merge(configuration(chosenPresets: ['baz', 'bang']));
expect(merged.chosenPresets, equals(['baz', 'bang']));
});
test('if both are defined, unions them', () {
- var merged = Configuration(chosenPresets: ['baz', 'bang'])
- .merge(Configuration(chosenPresets: ['qux']));
+ var merged = configuration(chosenPresets: ['baz', 'bang'])
+ .merge(configuration(chosenPresets: ['qux']));
expect(merged.chosenPresets, equals(['baz', 'bang', 'qux']));
});
});
group('for presets', () {
test('merges each nested configuration', () {
- var merged = Configuration(presets: {
- 'bang': Configuration(pauseAfterLoad: true),
- 'qux': Configuration(color: true)
- }).merge(Configuration(presets: {
- 'qux': Configuration(color: false),
- 'zap': Configuration(help: true)
+ var merged = configuration(presets: {
+ 'bang': configuration(pauseAfterLoad: true),
+ 'qux': configuration(color: true)
+ }).merge(configuration(presets: {
+ 'qux': configuration(color: false),
+ 'zap': configuration(help: true)
}));
expect(merged.presets['bang']!.pauseAfterLoad, isTrue);
@@ -190,68 +191,67 @@
});
test('automatically resolves a matching chosen preset', () {
- var configuration = Configuration(
- presets: {'foo': Configuration(color: true)},
+ var config = configuration(
+ presets: {'foo': configuration(color: true)},
chosenPresets: ['foo']);
- expect(configuration.presets, isEmpty);
- expect(configuration.chosenPresets, equals(['foo']));
- expect(configuration.knownPresets, equals(['foo']));
- expect(configuration.color, isTrue);
+ expect(config.presets, isEmpty);
+ expect(config.chosenPresets, equals(['foo']));
+ expect(config.knownPresets, equals(['foo']));
+ expect(config.color, isTrue);
});
test('resolves a chosen presets in order', () {
- var configuration = Configuration(presets: {
- 'foo': Configuration(color: true),
- 'bar': Configuration(color: false)
+ var config = configuration(presets: {
+ 'foo': configuration(color: true),
+ 'bar': configuration(color: false)
}, chosenPresets: [
'foo',
'bar'
]);
- expect(configuration.presets, isEmpty);
- expect(configuration.chosenPresets, equals(['foo', 'bar']));
- expect(configuration.knownPresets, unorderedEquals(['foo', 'bar']));
- expect(configuration.color, isFalse);
+ expect(config.presets, isEmpty);
+ expect(config.chosenPresets, equals(['foo', 'bar']));
+ expect(config.knownPresets, unorderedEquals(['foo', 'bar']));
+ expect(config.color, isFalse);
- configuration = Configuration(presets: {
- 'foo': Configuration(color: true),
- 'bar': Configuration(color: false)
+ config = configuration(presets: {
+ 'foo': configuration(color: true),
+ 'bar': configuration(color: false)
}, chosenPresets: [
'bar',
'foo'
]);
- expect(configuration.presets, isEmpty);
- expect(configuration.chosenPresets, equals(['bar', 'foo']));
- expect(configuration.knownPresets, unorderedEquals(['foo', 'bar']));
- expect(configuration.color, isTrue);
+ expect(config.presets, isEmpty);
+ expect(config.chosenPresets, equals(['bar', 'foo']));
+ expect(config.knownPresets, unorderedEquals(['foo', 'bar']));
+ expect(config.color, isTrue);
});
test('ignores inapplicable chosen presets', () {
- var configuration = Configuration(presets: {}, chosenPresets: ['baz']);
- expect(configuration.presets, isEmpty);
- expect(configuration.chosenPresets, equals(['baz']));
- expect(configuration.knownPresets, equals(isEmpty));
+ var config = configuration(presets: {}, chosenPresets: ['baz']);
+ expect(config.presets, isEmpty);
+ expect(config.chosenPresets, equals(['baz']));
+ expect(config.knownPresets, equals(isEmpty));
});
test('resolves presets through merging', () {
- var configuration =
- Configuration(presets: {'foo': Configuration(color: true)})
- .merge(Configuration(chosenPresets: ['foo']));
+ var config = configuration(presets: {'foo': configuration(color: true)})
+ .merge(configuration(chosenPresets: ['foo']));
- expect(configuration.presets, isEmpty);
- expect(configuration.chosenPresets, equals(['foo']));
- expect(configuration.knownPresets, equals(['foo']));
- expect(configuration.color, isTrue);
+ expect(config.presets, isEmpty);
+ expect(config.chosenPresets, equals(['foo']));
+ expect(config.knownPresets, equals(['foo']));
+ expect(config.color, isTrue);
});
test('preserves known presets through merging', () {
- var configuration = Configuration(
- presets: {'foo': Configuration(color: true)},
- chosenPresets: ['foo']).merge(Configuration());
+ var config = configuration(
+ presets: {'foo': configuration(color: true)},
+ chosenPresets: ['foo']).merge(configuration());
- expect(configuration.presets, isEmpty);
- expect(configuration.chosenPresets, equals(['foo']));
- expect(configuration.knownPresets, equals(['foo']));
- expect(configuration.color, isTrue);
+ expect(config.presets, isEmpty);
+ expect(config.chosenPresets, equals(['foo']));
+ expect(config.knownPresets, equals(['foo']));
+ expect(config.color, isTrue);
});
});
});
diff --git a/pkgs/test/test/runner/configuration/randomize_order_test.dart b/pkgs/test/test/runner/configuration/randomize_order_test.dart
index 984cd0f..adb853e 100644
--- a/pkgs/test/test/runner/configuration/randomize_order_test.dart
+++ b/pkgs/test/test/runner/configuration/randomize_order_test.dart
@@ -3,6 +3,7 @@
// BSD-style license that can be found in the LICENSE file.
@TestOn('vm')
+import 'dart:convert';
import 'package:test_descriptor/test_descriptor.dart' as d;
@@ -82,6 +83,43 @@
await test.shouldExit(0);
});
+ test('test shuffling can be disabled in dart_test.yml', () async {
+ await d
+ .file(
+ 'dart_test.yaml',
+ jsonEncode({
+ 'tags': {
+ 'doNotShuffle': {'allow_test_randomization': false}
+ }
+ }))
+ .create();
+
+ await d.file('test.dart', '''
+ @Tags(['doNotShuffle'])
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 1", () {});
+ test("test 2", () {});
+ test("test 3", () {});
+ test("test 4", () {});
+ }
+ ''').create();
+
+ var test =
+ await runTest(['test.dart', '--test-randomize-ordering-seed=987654']);
+ expect(
+ test.stdout,
+ containsInOrder([
+ '+0: test 1',
+ '+1: test 2',
+ '+2: test 3',
+ '+3: test 4',
+ '+4: All tests passed!'
+ ]));
+ await test.shouldExit(0);
+ });
+
test('shuffles each suite with the same seed', () async {
await d.file('1_test.dart', '''
import 'package:test/test.dart';
diff --git a/pkgs/test/test/runner/configuration/suite_test.dart b/pkgs/test/test/runner/configuration/suite_test.dart
index 8fa2069..bcd64a5 100644
--- a/pkgs/test/test/runner/configuration/suite_test.dart
+++ b/pkgs/test/test/runner/configuration/suite_test.dart
@@ -9,13 +9,14 @@
import 'package:test_api/src/backend/platform_selector.dart';
import 'package:test_api/src/backend/runtime.dart';
import 'package:test_core/src/runner/runtime_selection.dart';
-import 'package:test_core/src/runner/suite.dart';
+
+import '../../utils.dart';
void main() {
group('merge', () {
group('for most fields', () {
test('if neither is defined, preserves the default', () {
- var merged = SuiteConfiguration().merge(SuiteConfiguration());
+ var merged = suiteConfiguration().merge(suiteConfiguration());
expect(merged.jsTrace, isFalse);
expect(merged.runSkipped, isFalse);
expect(merged.precompiledPath, isNull);
@@ -23,12 +24,12 @@
});
test("if only the old configuration's is defined, uses it", () {
- var merged = SuiteConfiguration(
+ var merged = suiteConfiguration(
jsTrace: true,
runSkipped: true,
precompiledPath: '/tmp/js',
runtimes: [RuntimeSelection(Runtime.chrome.identifier)])
- .merge(SuiteConfiguration());
+ .merge(suiteConfiguration());
expect(merged.jsTrace, isTrue);
expect(merged.runSkipped, isTrue);
@@ -37,7 +38,7 @@
});
test("if only the configuration's is defined, uses it", () {
- var merged = SuiteConfiguration().merge(SuiteConfiguration(
+ var merged = suiteConfiguration().merge(suiteConfiguration(
jsTrace: true,
runSkipped: true,
precompiledPath: '/tmp/js',
@@ -52,12 +53,12 @@
test(
"if the two configurations conflict, uses the configuration's "
'values', () {
- var older = SuiteConfiguration(
+ var older = suiteConfiguration(
jsTrace: false,
runSkipped: true,
precompiledPath: '/tmp/js',
runtimes: [RuntimeSelection(Runtime.chrome.identifier)]);
- var newer = SuiteConfiguration(
+ var newer = suiteConfiguration(
jsTrace: true,
runSkipped: false,
precompiledPath: '../js',
@@ -73,16 +74,16 @@
group('for include and excludeTags', () {
test('if neither is defined, preserves the default', () {
- var merged = SuiteConfiguration().merge(SuiteConfiguration());
+ var merged = suiteConfiguration().merge(suiteConfiguration());
expect(merged.includeTags, equals(BooleanSelector.all));
expect(merged.excludeTags, equals(BooleanSelector.none));
});
test("if only the old configuration's is defined, uses it", () {
- var merged = SuiteConfiguration(
+ var merged = suiteConfiguration(
includeTags: BooleanSelector.parse('foo || bar'),
excludeTags: BooleanSelector.parse('baz || bang'))
- .merge(SuiteConfiguration());
+ .merge(suiteConfiguration());
expect(merged.includeTags, equals(BooleanSelector.parse('foo || bar')));
expect(
@@ -90,7 +91,7 @@
});
test("if only the configuration's is defined, uses it", () {
- var merged = SuiteConfiguration().merge(SuiteConfiguration(
+ var merged = suiteConfiguration().merge(suiteConfiguration(
includeTags: BooleanSelector.parse('foo || bar'),
excludeTags: BooleanSelector.parse('baz || bang')));
@@ -100,10 +101,10 @@
});
test('if both are defined, unions or intersects them', () {
- var older = SuiteConfiguration(
+ var older = suiteConfiguration(
includeTags: BooleanSelector.parse('foo || bar'),
excludeTags: BooleanSelector.parse('baz || bang'));
- var newer = SuiteConfiguration(
+ var newer = suiteConfiguration(
includeTags: BooleanSelector.parse('blip'),
excludeTags: BooleanSelector.parse('qux'));
var merged = older.merge(newer);
@@ -117,27 +118,27 @@
group('for sets', () {
test('if neither is defined, preserves the default', () {
- var merged = SuiteConfiguration().merge(SuiteConfiguration());
+ var merged = suiteConfiguration().merge(suiteConfiguration());
expect(merged.patterns, isEmpty);
});
test("if only the old configuration's is defined, uses it", () {
- var merged = SuiteConfiguration(patterns: ['beep', 'boop'])
- .merge(SuiteConfiguration());
+ var merged = suiteConfiguration(patterns: ['beep', 'boop'])
+ .merge(suiteConfiguration());
expect(merged.patterns, equals(['beep', 'boop']));
});
test("if only the configuration's is defined, uses it", () {
- var merged = SuiteConfiguration()
- .merge(SuiteConfiguration(patterns: ['beep', 'boop']));
+ var merged = suiteConfiguration()
+ .merge(suiteConfiguration(patterns: ['beep', 'boop']));
expect(merged.patterns, equals(['beep', 'boop']));
});
test('if both are defined, unions them', () {
- var older = SuiteConfiguration(patterns: ['beep', 'boop']);
- var newer = SuiteConfiguration(patterns: ['bonk']);
+ var older = suiteConfiguration(patterns: ['beep', 'boop']);
+ var newer = suiteConfiguration(patterns: ['bonk']);
var merged = older.merge(newer);
expect(merged.patterns, unorderedEquals(['beep', 'boop', 'bonk']));
@@ -146,25 +147,25 @@
group('for dart2jsArgs', () {
test('if neither is defined, preserves the default', () {
- var merged = SuiteConfiguration().merge(SuiteConfiguration());
+ var merged = suiteConfiguration().merge(suiteConfiguration());
expect(merged.dart2jsArgs, isEmpty);
});
test("if only the old configuration's is defined, uses it", () {
- var merged = SuiteConfiguration(dart2jsArgs: ['--foo', '--bar'])
- .merge(SuiteConfiguration());
+ var merged = suiteConfiguration(dart2jsArgs: ['--foo', '--bar'])
+ .merge(suiteConfiguration());
expect(merged.dart2jsArgs, equals(['--foo', '--bar']));
});
test("if only the configuration's is defined, uses it", () {
- var merged = SuiteConfiguration()
- .merge(SuiteConfiguration(dart2jsArgs: ['--foo', '--bar']));
+ var merged = suiteConfiguration()
+ .merge(suiteConfiguration(dart2jsArgs: ['--foo', '--bar']));
expect(merged.dart2jsArgs, equals(['--foo', '--bar']));
});
test('if both are defined, concatenates them', () {
- var older = SuiteConfiguration(dart2jsArgs: ['--foo', '--bar']);
- var newer = SuiteConfiguration(dart2jsArgs: ['--baz']);
+ var older = suiteConfiguration(dart2jsArgs: ['--foo', '--bar']);
+ var newer = suiteConfiguration(dart2jsArgs: ['--baz']);
var merged = older.merge(newer);
expect(merged.dart2jsArgs, equals(['--foo', '--bar', '--baz']));
});
@@ -172,21 +173,21 @@
group('for config maps', () {
test('merges each nested configuration', () {
- var merged = SuiteConfiguration(tags: {
+ var merged = suiteConfiguration(tags: {
BooleanSelector.parse('foo'):
- SuiteConfiguration(precompiledPath: 'path/'),
- BooleanSelector.parse('bar'): SuiteConfiguration(jsTrace: true)
+ suiteConfiguration(precompiledPath: 'path/'),
+ BooleanSelector.parse('bar'): suiteConfiguration(jsTrace: true)
}, onPlatform: {
PlatformSelector.parse('vm'):
- SuiteConfiguration(precompiledPath: 'path/'),
- PlatformSelector.parse('chrome'): SuiteConfiguration(jsTrace: true)
- }).merge(SuiteConfiguration(tags: {
- BooleanSelector.parse('bar'): SuiteConfiguration(jsTrace: false),
- BooleanSelector.parse('baz'): SuiteConfiguration(runSkipped: true)
+ suiteConfiguration(precompiledPath: 'path/'),
+ PlatformSelector.parse('chrome'): suiteConfiguration(jsTrace: true)
+ }).merge(suiteConfiguration(tags: {
+ BooleanSelector.parse('bar'): suiteConfiguration(jsTrace: false),
+ BooleanSelector.parse('baz'): suiteConfiguration(runSkipped: true)
}, onPlatform: {
- PlatformSelector.parse('chrome'): SuiteConfiguration(jsTrace: false),
+ PlatformSelector.parse('chrome'): suiteConfiguration(jsTrace: false),
PlatformSelector.parse('firefox'):
- SuiteConfiguration(runSkipped: true)
+ suiteConfiguration(runSkipped: true)
}));
expect(merged.tags[BooleanSelector.parse('foo')]!.precompiledPath,
diff --git a/pkgs/test/test/runner/loader_test.dart b/pkgs/test/test/runner/loader_test.dart
index 3469d4e..595514d 100644
--- a/pkgs/test/test/runner/loader_test.dart
+++ b/pkgs/test/test/runner/loader_test.dart
@@ -177,7 +177,7 @@
await runZoned(() async {
var suites = await _loader
.loadFile(p.join(d.sandbox, 'a_test.dart'),
- SuiteConfiguration(retry: numRetries))
+ suiteConfiguration(retry: numRetries))
.toList();
expect(suites, hasLength(1));
var loadSuite = suites.first;
diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart
index 4b04134..123ff0a 100644
--- a/pkgs/test/test/utils.dart
+++ b/pkgs/test/test/utils.dart
@@ -5,22 +5,29 @@
import 'dart:async';
import 'dart:collection';
+import 'package:boolean_selector/boolean_selector.dart';
+import 'package:glob/glob.dart';
import 'package:test_api/src/backend/declarer.dart';
import 'package:test_api/src/backend/group.dart';
import 'package:test_api/src/backend/group_entry.dart';
import 'package:test_api/src/backend/invoker.dart';
import 'package:test_api/src/backend/live_test.dart';
import 'package:test_api/src/backend/metadata.dart';
+import 'package:test_api/src/backend/platform_selector.dart';
import 'package:test_api/src/backend/runtime.dart';
import 'package:test_api/src/backend/state.dart';
import 'package:test_api/src/backend/suite.dart';
import 'package:test_api/src/backend/suite_platform.dart';
import 'package:test_core/src/runner/application_exception.dart';
-import 'package:test_core/src/runner/suite.dart';
+import 'package:test_core/src/runner/configuration.dart';
+import 'package:test_core/src/runner/configuration/custom_runtime.dart';
+import 'package:test_core/src/runner/configuration/runtime_settings.dart';
import 'package:test_core/src/runner/engine.dart';
-import 'package:test_core/src/runner/plugin/environment.dart';
import 'package:test_core/src/runner/load_suite.dart';
+import 'package:test_core/src/runner/plugin/environment.dart';
import 'package:test_core/src/runner/runner_suite.dart';
+import 'package:test_core/src/runner/runtime_selection.dart';
+import 'package:test_core/src/runner/suite.dart';
import 'package:test/test.dart';
/// A dummy suite platform to use for testing suites.
@@ -193,7 +200,7 @@
return Engine.withSuites([
RunnerSuite(
const PluginEnvironment(),
- SuiteConfiguration(runSkipped: runSkipped),
+ SuiteConfiguration.runSkipped(runSkipped),
declarer.build(),
suitePlatform)
], coverage: coverage);
@@ -206,3 +213,143 @@
/// Returns a [LoadSuite] with a default configuration.
LoadSuite loadSuite(String name, FutureOr<RunnerSuite> Function() body) =>
LoadSuite(name, SuiteConfiguration.empty, suitePlatform, body);
+
+SuiteConfiguration suiteConfiguration(
+ {bool? allowTestRandomization,
+ bool? jsTrace,
+ bool? runSkipped,
+ Iterable<String>? dart2jsArgs,
+ String? precompiledPath,
+ Iterable<Pattern>? patterns,
+ Iterable<RuntimeSelection>? runtimes,
+ BooleanSelector? includeTags,
+ BooleanSelector? excludeTags,
+ Map<BooleanSelector, SuiteConfiguration>? tags,
+ Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+
+ // Test-level configuration
+ Timeout? timeout,
+ bool? verboseTrace,
+ bool? chainStackTraces,
+ bool? skip,
+ int? retry,
+ String? skipReason,
+ PlatformSelector? testOn,
+ Iterable<String>? addTags}) =>
+ SuiteConfiguration(
+ allowTestRandomization: allowTestRandomization,
+ jsTrace: jsTrace,
+ runSkipped: runSkipped,
+ dart2jsArgs: dart2jsArgs,
+ precompiledPath: precompiledPath,
+ patterns: patterns,
+ runtimes: runtimes,
+ includeTags: includeTags,
+ excludeTags: excludeTags,
+ tags: tags,
+ onPlatform: onPlatform,
+ timeout: timeout,
+ verboseTrace: verboseTrace,
+ chainStackTraces: chainStackTraces,
+ skip: skip,
+ retry: retry,
+ skipReason: skipReason,
+ testOn: testOn,
+ addTags: addTags);
+
+Configuration configuration(
+ {bool? help,
+ String? customHtmlTemplatePath,
+ bool? version,
+ bool? pauseAfterLoad,
+ bool? debug,
+ bool? color,
+ String? configurationPath,
+ String? dart2jsPath,
+ String? reporter,
+ Map<String, String>? fileReporters,
+ String? coverage,
+ int? pubServePort,
+ int? concurrency,
+ int? shardIndex,
+ int? totalShards,
+ Iterable<String>? paths,
+ Iterable<String>? foldTraceExcept,
+ Iterable<String>? foldTraceOnly,
+ Glob? filename,
+ Iterable<String>? chosenPresets,
+ Map<String, Configuration>? presets,
+ Map<String, RuntimeSettings>? overrideRuntimes,
+ Map<String, CustomRuntime>? defineRuntimes,
+ bool? noRetry,
+ bool? useDataIsolateStrategy,
+
+ // Suite-level configuration
+ bool? allowTestRandomization,
+ bool? jsTrace,
+ bool? runSkipped,
+ Iterable<String>? dart2jsArgs,
+ String? precompiledPath,
+ Iterable<Pattern>? patterns,
+ Iterable<RuntimeSelection>? runtimes,
+ BooleanSelector? includeTags,
+ BooleanSelector? excludeTags,
+ Map<BooleanSelector, SuiteConfiguration>? tags,
+ Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+ int? testRandomizeOrderingSeed,
+
+ // Test-level configuration
+ Timeout? timeout,
+ bool? verboseTrace,
+ bool? chainStackTraces,
+ bool? skip,
+ int? retry,
+ String? skipReason,
+ PlatformSelector? testOn,
+ Iterable<String>? addTags}) =>
+ Configuration(
+ help: help,
+ customHtmlTemplatePath: customHtmlTemplatePath,
+ version: version,
+ pauseAfterLoad: pauseAfterLoad,
+ debug: debug,
+ color: color,
+ configurationPath: configurationPath,
+ dart2jsPath: dart2jsPath,
+ reporter: reporter,
+ fileReporters: fileReporters,
+ coverage: coverage,
+ pubServePort: pubServePort,
+ concurrency: concurrency,
+ shardIndex: shardIndex,
+ totalShards: totalShards,
+ paths: paths,
+ foldTraceExcept: foldTraceExcept,
+ foldTraceOnly: foldTraceOnly,
+ filename: filename,
+ chosenPresets: chosenPresets,
+ presets: presets,
+ overrideRuntimes: overrideRuntimes,
+ defineRuntimes: defineRuntimes,
+ noRetry: noRetry,
+ useDataIsolateStrategy: useDataIsolateStrategy,
+ allowTestRandomization: allowTestRandomization,
+ jsTrace: jsTrace,
+ runSkipped: runSkipped,
+ dart2jsArgs: dart2jsArgs,
+ precompiledPath: precompiledPath,
+ patterns: patterns,
+ runtimes: runtimes,
+ includeTags: includeTags,
+ excludeTags: excludeTags,
+ tags: tags,
+ onPlatform: onPlatform,
+ testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ timeout: timeout,
+ verboseTrace: verboseTrace,
+ chainStackTraces: chainStackTraces,
+ skip: skip,
+ retry: retry,
+ skipReason: skipReason,
+ testOn: testOn,
+ addTags: addTags);
diff --git a/pkgs/test_api/test/utils.dart b/pkgs/test_api/test/utils.dart
index 18981d5..22888a7 100644
--- a/pkgs/test_api/test/utils.dart
+++ b/pkgs/test_api/test/utils.dart
@@ -181,7 +181,7 @@
return Engine.withSuites([
RunnerSuite(
const PluginEnvironment(),
- SuiteConfiguration(runSkipped: runSkipped),
+ SuiteConfiguration.runSkipped(runSkipped),
declarer.build(),
suitePlatform)
]);
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index 2d33d44..95aab63 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -1,6 +1,9 @@
-## 0.3.30-dev
+## 0.4.0-dev
-* Remove support for `FORCE_TEST_EXIT`.
+* **BREAKING**: All parameters to the `SuiteConfiguration` and `Configuration`
+ constructors are now required. Some specialized constructors have been added
+ for the common cases where a subset are intended to be provided.
+* **BREAKING**: Remove support for `FORCE_TEST_EXIT`.
* Report incomplete tests as errors in the JSON reporter when the run is
canceled early.
* Don't log the --test-randomization-ordering-seed if using the json reporter.
diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart
index ae21042..0f9e333 100644
--- a/pkgs/test_core/lib/src/runner/configuration.dart
+++ b/pkgs/test_core/lib/src/runner/configuration.dart
@@ -36,7 +36,7 @@
///
/// Using this is slightly more efficient than manually constructing a new
/// configuration with no arguments.
- static final empty = Configuration._();
+ static final empty = Configuration._unsafe();
/// The usage string for the command-line arguments.
static String get usage => args.usage;
@@ -202,7 +202,7 @@
///
/// The current configuration is set using [asCurrent].
static Configuration get current =>
- Zone.current[_currentKey] as Configuration? ?? Configuration();
+ Zone.current[_currentKey] as Configuration? ?? Configuration._unsafe();
/// Parses the configuration from [args].
///
@@ -236,54 +236,55 @@
parse(content, global: global, sourceUrl: sourceUrl);
factory Configuration(
- {bool? help,
- String? customHtmlTemplatePath,
- bool? version,
- bool? pauseAfterLoad,
- bool? debug,
- bool? color,
- String? configurationPath,
- String? dart2jsPath,
- String? reporter,
- Map<String, String>? fileReporters,
- String? coverage,
- int? pubServePort,
- int? concurrency,
- int? shardIndex,
- int? totalShards,
- Iterable<String>? paths,
- Iterable<String>? foldTraceExcept,
- Iterable<String>? foldTraceOnly,
- Glob? filename,
- Iterable<String>? chosenPresets,
- Map<String, Configuration>? presets,
- Map<String, RuntimeSettings>? overrideRuntimes,
- Map<String, CustomRuntime>? defineRuntimes,
- bool? noRetry,
- bool? useDataIsolateStrategy,
+ {required bool? help,
+ required String? customHtmlTemplatePath,
+ required bool? version,
+ required bool? pauseAfterLoad,
+ required bool? debug,
+ required bool? color,
+ required String? configurationPath,
+ required String? dart2jsPath,
+ required String? reporter,
+ required Map<String, String>? fileReporters,
+ required String? coverage,
+ required int? pubServePort,
+ required int? concurrency,
+ required int? shardIndex,
+ required int? totalShards,
+ required Iterable<String>? paths,
+ required Iterable<String>? foldTraceExcept,
+ required Iterable<String>? foldTraceOnly,
+ required Glob? filename,
+ required Iterable<String>? chosenPresets,
+ required Map<String, Configuration>? presets,
+ required Map<String, RuntimeSettings>? overrideRuntimes,
+ required Map<String, CustomRuntime>? defineRuntimes,
+ required bool? noRetry,
+ required bool? useDataIsolateStrategy,
+ required int? testRandomizeOrderingSeed,
// Suite-level configuration
- bool? jsTrace,
- bool? runSkipped,
- Iterable<String>? dart2jsArgs,
- String? precompiledPath,
- Iterable<Pattern>? patterns,
- Iterable<RuntimeSelection>? runtimes,
- BooleanSelector? includeTags,
- BooleanSelector? excludeTags,
- Map<BooleanSelector, SuiteConfiguration>? tags,
- Map<PlatformSelector, SuiteConfiguration>? onPlatform,
- int? testRandomizeOrderingSeed,
+ required bool? allowTestRandomization,
+ required bool? jsTrace,
+ required bool? runSkipped,
+ required Iterable<String>? dart2jsArgs,
+ required String? precompiledPath,
+ required Iterable<Pattern>? patterns,
+ required Iterable<RuntimeSelection>? runtimes,
+ required BooleanSelector? includeTags,
+ required BooleanSelector? excludeTags,
+ required Map<BooleanSelector, SuiteConfiguration>? tags,
+ required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
// Test-level configuration
- Timeout? timeout,
- bool? verboseTrace,
- bool? chainStackTraces,
- bool? skip,
- int? retry,
- String? skipReason,
- PlatformSelector? testOn,
- Iterable<String>? addTags}) {
+ required Timeout? timeout,
+ required bool? verboseTrace,
+ required bool? chainStackTraces,
+ required bool? skip,
+ required int? retry,
+ required String? skipReason,
+ required PlatformSelector? testOn,
+ required Iterable<String>? addTags}) {
var chosenPresetSet = chosenPresets?.toSet();
var configuration = Configuration._(
help: help,
@@ -313,6 +314,7 @@
useDataIsolateStrategy: useDataIsolateStrategy,
testRandomizeOrderingSeed: testRandomizeOrderingSeed,
suiteDefaults: SuiteConfiguration(
+ allowTestRandomization: allowTestRandomization,
jsTrace: jsTrace,
runSkipped: runSkipped,
dart2jsArgs: dart2jsArgs,
@@ -336,6 +338,344 @@
return configuration._resolvePresets();
}
+ /// A constructor that doesn't require all of its options to be passed.
+ ///
+ /// This should only be used in situations where you really only want to
+ /// configure a specific restricted set of options.
+ factory Configuration._unsafe(
+ {bool? help,
+ String? customHtmlTemplatePath,
+ bool? version,
+ bool? pauseAfterLoad,
+ bool? debug,
+ bool? color,
+ String? configurationPath,
+ String? dart2jsPath,
+ String? reporter,
+ Map<String, String>? fileReporters,
+ String? coverage,
+ int? pubServePort,
+ int? concurrency,
+ int? shardIndex,
+ int? totalShards,
+ Iterable<String>? paths,
+ Iterable<String>? foldTraceExcept,
+ Iterable<String>? foldTraceOnly,
+ Glob? filename,
+ Iterable<String>? chosenPresets,
+ Map<String, Configuration>? presets,
+ Map<String, RuntimeSettings>? overrideRuntimes,
+ Map<String, CustomRuntime>? defineRuntimes,
+ bool? noRetry,
+ bool? useDataIsolateStrategy,
+
+ // Suite-level configuration
+ bool? allowTestRandomization,
+ bool? jsTrace,
+ bool? runSkipped,
+ Iterable<String>? dart2jsArgs,
+ String? precompiledPath,
+ Iterable<Pattern>? patterns,
+ Iterable<RuntimeSelection>? runtimes,
+ BooleanSelector? includeTags,
+ BooleanSelector? excludeTags,
+ Map<BooleanSelector, SuiteConfiguration>? tags,
+ Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+ int? testRandomizeOrderingSeed,
+
+ // Test-level configuration
+ Timeout? timeout,
+ bool? verboseTrace,
+ bool? chainStackTraces,
+ bool? skip,
+ int? retry,
+ String? skipReason,
+ PlatformSelector? testOn,
+ Iterable<String>? addTags}) =>
+ Configuration(
+ help: help,
+ customHtmlTemplatePath: customHtmlTemplatePath,
+ version: version,
+ pauseAfterLoad: pauseAfterLoad,
+ debug: debug,
+ color: color,
+ configurationPath: configurationPath,
+ dart2jsPath: dart2jsPath,
+ reporter: reporter,
+ fileReporters: fileReporters,
+ coverage: coverage,
+ pubServePort: pubServePort,
+ concurrency: concurrency,
+ shardIndex: shardIndex,
+ totalShards: totalShards,
+ paths: paths,
+ foldTraceExcept: foldTraceExcept,
+ foldTraceOnly: foldTraceOnly,
+ filename: filename,
+ chosenPresets: chosenPresets,
+ presets: presets,
+ overrideRuntimes: overrideRuntimes,
+ defineRuntimes: defineRuntimes,
+ noRetry: noRetry,
+ useDataIsolateStrategy: useDataIsolateStrategy,
+ allowTestRandomization: allowTestRandomization,
+ jsTrace: jsTrace,
+ runSkipped: runSkipped,
+ dart2jsArgs: dart2jsArgs,
+ precompiledPath: precompiledPath,
+ patterns: patterns,
+ runtimes: runtimes,
+ includeTags: includeTags,
+ excludeTags: excludeTags,
+ tags: tags,
+ onPlatform: onPlatform,
+ testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ timeout: timeout,
+ verboseTrace: verboseTrace,
+ chainStackTraces: chainStackTraces,
+ skip: skip,
+ retry: retry,
+ skipReason: skipReason,
+ testOn: testOn,
+ addTags: addTags);
+
+ /// Configuration limited to the globally configurable test config.
+ factory Configuration.globalTest({
+ required bool? verboseTrace,
+ required bool? jsTrace,
+ required Timeout? timeout,
+ required Map<String, Configuration>? presets,
+ required bool? chainStackTraces,
+ required Iterable<String>? foldTraceExcept,
+ required Iterable<String>? foldTraceOnly,
+ }) =>
+ Configuration(
+ foldTraceExcept: foldTraceExcept,
+ foldTraceOnly: foldTraceOnly,
+ jsTrace: jsTrace,
+ timeout: timeout,
+ verboseTrace: verboseTrace,
+ chainStackTraces: chainStackTraces,
+ help: null,
+ customHtmlTemplatePath: null,
+ version: null,
+ pauseAfterLoad: null,
+ debug: null,
+ color: null,
+ configurationPath: null,
+ dart2jsPath: null,
+ reporter: null,
+ fileReporters: null,
+ coverage: null,
+ pubServePort: null,
+ concurrency: null,
+ shardIndex: null,
+ totalShards: null,
+ paths: null,
+ filename: null,
+ chosenPresets: null,
+ presets: presets,
+ overrideRuntimes: null,
+ defineRuntimes: null,
+ noRetry: null,
+ useDataIsolateStrategy: null,
+ testRandomizeOrderingSeed: null,
+ allowTestRandomization: null,
+ runSkipped: null,
+ dart2jsArgs: null,
+ precompiledPath: null,
+ patterns: null,
+ runtimes: null,
+ includeTags: null,
+ excludeTags: null,
+ tags: null,
+ onPlatform: null,
+ skip: null,
+ retry: null,
+ skipReason: null,
+ testOn: null,
+ addTags: null,
+ );
+
+ /// Configuration limited to the locally configurable test config.
+ factory Configuration.localTest(
+ {required bool? skip,
+ required int? retry,
+ required String? skipReason,
+ required PlatformSelector? testOn,
+ required Iterable<String>? addTags,
+ required bool? allowTestRandomization}) =>
+ Configuration(
+ allowTestRandomization: allowTestRandomization,
+ skip: skip,
+ retry: retry,
+ skipReason: skipReason,
+ testOn: testOn,
+ addTags: addTags,
+ help: null,
+ customHtmlTemplatePath: null,
+ version: null,
+ pauseAfterLoad: null,
+ debug: null,
+ color: null,
+ configurationPath: null,
+ dart2jsPath: null,
+ reporter: null,
+ fileReporters: null,
+ coverage: null,
+ pubServePort: null,
+ concurrency: null,
+ shardIndex: null,
+ totalShards: null,
+ paths: null,
+ foldTraceExcept: null,
+ foldTraceOnly: null,
+ filename: null,
+ chosenPresets: null,
+ presets: null,
+ overrideRuntimes: null,
+ defineRuntimes: null,
+ noRetry: null,
+ useDataIsolateStrategy: null,
+ testRandomizeOrderingSeed: null,
+ jsTrace: null,
+ runSkipped: null,
+ dart2jsArgs: null,
+ precompiledPath: null,
+ patterns: null,
+ runtimes: null,
+ includeTags: null,
+ excludeTags: null,
+ tags: null,
+ onPlatform: null,
+ timeout: null,
+ verboseTrace: null,
+ chainStackTraces: null,
+ );
+
+ /// Configuration options limited to the global runner config.
+ factory Configuration.globalRunner(
+ {required bool? pauseAfterLoad,
+ required String? customHtmlTemplatePath,
+ required bool? runSkipped,
+ required String? reporter,
+ required Map<String, String>? fileReporters,
+ required int? concurrency,
+ required Iterable<RuntimeSelection>? runtimes,
+ required Iterable<String>? chosenPresets,
+ required Map<String, RuntimeSettings>? overrideRuntimes}) =>
+ Configuration(
+ customHtmlTemplatePath: customHtmlTemplatePath,
+ pauseAfterLoad: pauseAfterLoad,
+ runSkipped: runSkipped,
+ reporter: reporter,
+ fileReporters: fileReporters,
+ concurrency: concurrency,
+ runtimes: runtimes,
+ chosenPresets: chosenPresets,
+ overrideRuntimes: overrideRuntimes,
+ help: null,
+ version: null,
+ debug: null,
+ color: null,
+ configurationPath: null,
+ dart2jsPath: null,
+ coverage: null,
+ pubServePort: null,
+ shardIndex: null,
+ totalShards: null,
+ paths: null,
+ foldTraceExcept: null,
+ foldTraceOnly: null,
+ filename: null,
+ presets: null,
+ defineRuntimes: null,
+ noRetry: null,
+ useDataIsolateStrategy: null,
+ testRandomizeOrderingSeed: null,
+ allowTestRandomization: null,
+ jsTrace: null,
+ dart2jsArgs: null,
+ precompiledPath: null,
+ patterns: null,
+ includeTags: null,
+ excludeTags: null,
+ tags: null,
+ onPlatform: null,
+ timeout: null,
+ verboseTrace: null,
+ chainStackTraces: null,
+ skip: null,
+ retry: null,
+ skipReason: null,
+ testOn: null,
+ addTags: null,
+ );
+
+ /// Configuration options limited to the local runner config.
+ factory Configuration.localRunner(
+ {required int? pubServePort,
+ required Iterable<Pattern>? patterns,
+ required Iterable<String>? paths,
+ required Glob? filename,
+ required BooleanSelector? includeTags,
+ required BooleanSelector? excludeTags,
+ required Map<String, CustomRuntime>? defineRuntimes}) =>
+ Configuration(
+ pubServePort: pubServePort,
+ patterns: patterns,
+ paths: paths,
+ filename: filename,
+ includeTags: includeTags,
+ excludeTags: excludeTags,
+ defineRuntimes: defineRuntimes,
+ help: null,
+ customHtmlTemplatePath: null,
+ version: null,
+ pauseAfterLoad: null,
+ debug: null,
+ color: null,
+ configurationPath: null,
+ dart2jsPath: null,
+ reporter: null,
+ fileReporters: null,
+ coverage: null,
+ concurrency: null,
+ shardIndex: null,
+ totalShards: null,
+ foldTraceExcept: null,
+ foldTraceOnly: null,
+ chosenPresets: null,
+ presets: null,
+ overrideRuntimes: null,
+ noRetry: null,
+ useDataIsolateStrategy: null,
+ testRandomizeOrderingSeed: null,
+ allowTestRandomization: null,
+ jsTrace: null,
+ runSkipped: null,
+ dart2jsArgs: null,
+ precompiledPath: null,
+ runtimes: null,
+ tags: null,
+ onPlatform: null,
+ timeout: null,
+ verboseTrace: null,
+ chainStackTraces: null,
+ skip: null,
+ retry: null,
+ skipReason: null,
+ testOn: null,
+ addTags: null);
+
+ /// A specialized constructor for configuring only `onPlatform`.
+ factory Configuration.onPlatform(
+ Map<PlatformSelector, SuiteConfiguration> onPlatform) =>
+ Configuration._unsafe(onPlatform: onPlatform);
+
+ factory Configuration.tags(Map<BooleanSelector, SuiteConfiguration> tags) =>
+ Configuration._unsafe(tags: tags);
+
static Map<String, Configuration>? _withChosenPresets(
Map<String, Configuration>? map, Set<String>? chosenPresets) {
if (map == null || chosenPresets == null) return map;
@@ -349,33 +689,33 @@
///
/// Unlike [new Configuration], this assumes [presets] is already resolved.
Configuration._(
- {bool? help,
- String? customHtmlTemplatePath,
- bool? version,
- bool? pauseAfterLoad,
- bool? debug,
- bool? color,
- String? configurationPath,
- String? dart2jsPath,
- String? reporter,
- Map<String, String>? fileReporters,
- this.coverage,
- int? pubServePort,
- int? concurrency,
- this.shardIndex,
- this.totalShards,
- Iterable<String>? paths,
- Iterable<String>? foldTraceExcept,
- Iterable<String>? foldTraceOnly,
- Glob? filename,
- Iterable<String>? chosenPresets,
- Map<String, Configuration>? presets,
- Map<String, RuntimeSettings>? overrideRuntimes,
- Map<String, CustomRuntime>? defineRuntimes,
- bool? noRetry,
- bool? useDataIsolateStrategy,
- this.testRandomizeOrderingSeed,
- SuiteConfiguration? suiteDefaults})
+ {required bool? help,
+ required String? customHtmlTemplatePath,
+ required bool? version,
+ required bool? pauseAfterLoad,
+ required bool? debug,
+ required bool? color,
+ required String? configurationPath,
+ required String? dart2jsPath,
+ required String? reporter,
+ required Map<String, String>? fileReporters,
+ required this.coverage,
+ required int? pubServePort,
+ required int? concurrency,
+ required this.shardIndex,
+ required this.totalShards,
+ required Iterable<String>? paths,
+ required Iterable<String>? foldTraceExcept,
+ required Iterable<String>? foldTraceOnly,
+ required Glob? filename,
+ required Iterable<String>? chosenPresets,
+ required Map<String, Configuration>? presets,
+ required Map<String, RuntimeSettings>? overrideRuntimes,
+ required Map<String, CustomRuntime>? defineRuntimes,
+ required bool? noRetry,
+ required bool? useDataIsolateStrategy,
+ required this.testRandomizeOrderingSeed,
+ required SuiteConfiguration? suiteDefaults})
: _help = help,
customHtmlTemplatePath = customHtmlTemplatePath,
_version = version,
@@ -402,7 +742,7 @@
_useDataIsolateStrategy = useDataIsolateStrategy,
suiteDefaults = pauseAfterLoad == true
? suiteDefaults?.change(timeout: Timeout.none) ??
- SuiteConfiguration(timeout: Timeout.none)
+ SuiteConfiguration.timeout(Timeout.none)
: suiteDefaults ?? SuiteConfiguration.empty {
if (_filename != null && _filename!.context.style != p.style) {
throw ArgumentError(
@@ -423,7 +763,35 @@
/// [SuiteConfiguration].
factory Configuration.fromSuiteConfiguration(
SuiteConfiguration suiteConfig) =>
- Configuration._(suiteDefaults: suiteConfig);
+ Configuration._(
+ suiteDefaults: suiteConfig,
+ help: null,
+ customHtmlTemplatePath: null,
+ version: null,
+ pauseAfterLoad: null,
+ debug: null,
+ color: null,
+ configurationPath: null,
+ dart2jsPath: null,
+ reporter: null,
+ fileReporters: null,
+ coverage: null,
+ pubServePort: null,
+ concurrency: null,
+ shardIndex: null,
+ totalShards: null,
+ paths: null,
+ foldTraceExcept: null,
+ foldTraceOnly: null,
+ filename: null,
+ chosenPresets: null,
+ presets: null,
+ overrideRuntimes: null,
+ defineRuntimes: null,
+ noRetry: null,
+ useDataIsolateStrategy: null,
+ testRandomizeOrderingSeed: null,
+ );
/// Returns an unmodifiable copy of [input].
///
@@ -509,6 +877,7 @@
other.customHtmlTemplatePath ?? customHtmlTemplatePath,
version: other._version ?? _version,
pauseAfterLoad: other._pauseAfterLoad ?? _pauseAfterLoad,
+ debug: other._debug ?? _debug,
color: other._color ?? _color,
configurationPath: other._configurationPath ?? _configurationPath,
dart2jsPath: other._dart2jsPath ?? _dart2jsPath,
@@ -556,11 +925,13 @@
String? customHtmlTemplatePath,
bool? version,
bool? pauseAfterLoad,
+ bool? debug,
bool? color,
String? configurationPath,
String? dart2jsPath,
String? reporter,
Map<String, String>? fileReporters,
+ String? coverage,
int? pubServePort,
int? concurrency,
int? shardIndex,
@@ -603,11 +974,13 @@
customHtmlTemplatePath ?? this.customHtmlTemplatePath,
version: version ?? _version,
pauseAfterLoad: pauseAfterLoad ?? _pauseAfterLoad,
+ debug: debug ?? _debug,
color: color ?? _color,
configurationPath: configurationPath ?? _configurationPath,
dart2jsPath: dart2jsPath ?? _dart2jsPath,
reporter: reporter ?? _reporter,
fileReporters: fileReporters ?? this.fileReporters,
+ coverage: coverage ?? this.coverage,
pubServePort: pubServePort ?? pubServeUrl?.port,
concurrency: concurrency ?? _concurrency,
shardIndex: shardIndex ?? this.shardIndex,
@@ -623,6 +996,8 @@
noRetry: noRetry ?? _noRetry,
useDataIsolateStrategy:
useDataIsolateStrategy ?? _useDataIsolateStrategy,
+ testRandomizeOrderingSeed:
+ testRandomizeOrderingSeed ?? this.testRandomizeOrderingSeed,
suiteDefaults: suiteDefaults.change(
jsTrace: jsTrace,
runSkipped: runSkipped,
diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart
index e81c632..87c32ac 100644
--- a/pkgs/test_core/lib/src/runner/configuration/args.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/args.dart
@@ -274,7 +274,23 @@
excludeTags: excludeTags,
noRetry: _ifParsed('no-retry'),
useDataIsolateStrategy: _ifParsed('use-data-isolate-strategy'),
- testRandomizeOrderingSeed: testRandomizeOrderingSeed);
+ testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ // Config that isn't supported on the command line
+ addTags: null,
+ allowTestRandomization: null,
+ customHtmlTemplatePath: null,
+ defineRuntimes: null,
+ filename: null,
+ foldTraceExcept: null,
+ foldTraceOnly: null,
+ onPlatform: null,
+ overrideRuntimes: null,
+ presets: null,
+ retry: null,
+ skip: null,
+ skipReason: null,
+ testOn: null,
+ tags: null);
}
/// Returns the parsed option for [name], or `null` if none was parsed.
diff --git a/pkgs/test_core/lib/src/runner/configuration/load.dart b/pkgs/test_core/lib/src/runner/configuration/load.dart
index 57084e1..888869d 100644
--- a/pkgs/test_core/lib/src/runner/configuration/load.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/load.dart
@@ -147,7 +147,7 @@
key: (keyNode) => _parseIdentifierLike(keyNode, 'presets key'),
value: (valueNode) => _nestedConfig(valueNode, 'presets value'));
- var config = Configuration(
+ var config = Configuration.globalTest(
verboseTrace: verboseTrace,
jsTrace: jsTrace,
timeout: timeout,
@@ -156,7 +156,7 @@
foldTraceExcept: foldStackFrames['except'],
foldTraceOnly: foldStackFrames['only'])
.merge(_extractPresets<PlatformSelector>(
- onPlatform, (map) => Configuration(onPlatform: map)));
+ onPlatform, (map) => Configuration.onPlatform(map)));
var osConfig = onOS[currentOS];
return osConfig == null ? config : config.merge(osConfig);
@@ -174,6 +174,7 @@
_disallow('test_on');
_disallow('add_tags');
_disallow('tags');
+ _disallow('allow_test_randomization');
return Configuration.empty;
}
@@ -201,14 +202,17 @@
var retry = _getNonNegativeInt('retry');
- return Configuration(
+ var allowTestRandomization = _getBool('allow_test_randomization');
+
+ return Configuration.localTest(
skip: skip,
retry: retry,
skipReason: skipReason,
testOn: testOn,
- addTags: addTags)
+ addTags: addTags,
+ allowTestRandomization: allowTestRandomization)
.merge(_extractPresets<BooleanSelector>(
- tags, (map) => Configuration(tags: map)));
+ tags, (map) => Configuration.tags(map)));
}
/// Loads runner configuration that's allowed in the global configuration
@@ -272,7 +276,7 @@
var customHtmlTemplatePath = _getString('custom_html_template_path');
- return Configuration(
+ return Configuration.globalRunner(
pauseAfterLoad: pauseAfterLoad,
customHtmlTemplatePath: customHtmlTemplatePath,
runSkipped: runSkipped,
@@ -354,7 +358,7 @@
var defineRuntimes = _loadDefineRuntimes();
- return Configuration(
+ return Configuration.localRunner(
pubServePort: pubServePort,
patterns: patterns,
paths: paths,
diff --git a/pkgs/test_core/lib/src/runner/engine.dart b/pkgs/test_core/lib/src/runner/engine.dart
index 359374a..590e6a5 100644
--- a/pkgs/test_core/lib/src/runner/engine.dart
+++ b/pkgs/test_core/lib/src/runner/engine.dart
@@ -307,7 +307,8 @@
if (!_closed && setUpAllSucceeded) {
// shuffle the group entries
var entries = group.entries.toList();
- if (testRandomizeOrderingSeed != null &&
+ if (suiteConfig.allowTestRandomization &&
+ testRandomizeOrderingSeed != null &&
testRandomizeOrderingSeed! > 0) {
entries.shuffle(Random(testRandomizeOrderingSeed));
}
diff --git a/pkgs/test_core/lib/src/runner/suite.dart b/pkgs/test_core/lib/src/runner/suite.dart
index 7dd567b..520887f 100644
--- a/pkgs/test_core/lib/src/runner/suite.dart
+++ b/pkgs/test_core/lib/src/runner/suite.dart
@@ -23,7 +23,23 @@
///
/// Using this is slightly more efficient than manually constructing a new
/// configuration with no arguments.
- static final empty = SuiteConfiguration._();
+ static final empty = SuiteConfiguration._(
+ allowTestRandomization: null,
+ jsTrace: null,
+ runSkipped: null,
+ dart2jsArgs: null,
+ precompiledPath: null,
+ patterns: null,
+ runtimes: null,
+ includeTags: null,
+ excludeTags: null,
+ tags: null,
+ onPlatform: null,
+ metadata: null);
+
+ /// Whether test randomization should be allowed for this test.
+ bool get allowTestRandomization => _allowTestRandomization ?? true;
+ final bool? _allowTestRandomization;
/// Whether JavaScript stack traces should be left as-is or converted to
/// Dart-like traces.
@@ -125,27 +141,29 @@
}
factory SuiteConfiguration(
- {bool? jsTrace,
- bool? runSkipped,
- Iterable<String>? dart2jsArgs,
- String? precompiledPath,
- Iterable<Pattern>? patterns,
- Iterable<RuntimeSelection>? runtimes,
- BooleanSelector? includeTags,
- BooleanSelector? excludeTags,
- Map<BooleanSelector, SuiteConfiguration>? tags,
- Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+ {required bool? allowTestRandomization,
+ required bool? jsTrace,
+ required bool? runSkipped,
+ required Iterable<String>? dart2jsArgs,
+ required String? precompiledPath,
+ required Iterable<Pattern>? patterns,
+ required Iterable<RuntimeSelection>? runtimes,
+ required BooleanSelector? includeTags,
+ required BooleanSelector? excludeTags,
+ required Map<BooleanSelector, SuiteConfiguration>? tags,
+ required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
// Test-level configuration
- Timeout? timeout,
- bool? verboseTrace,
- bool? chainStackTraces,
- bool? skip,
- int? retry,
- String? skipReason,
- PlatformSelector? testOn,
- Iterable<String>? addTags}) {
+ required Timeout? timeout,
+ required bool? verboseTrace,
+ required bool? chainStackTraces,
+ required bool? skip,
+ required int? retry,
+ required String? skipReason,
+ required PlatformSelector? testOn,
+ required Iterable<String>? addTags}) {
var config = SuiteConfiguration._(
+ allowTestRandomization: allowTestRandomization,
jsTrace: jsTrace,
runSkipped: runSkipped,
dart2jsArgs: dart2jsArgs,
@@ -168,23 +186,84 @@
return config._resolveTags();
}
+ /// A constructor that doesn't require all of its options to be passed.
+ ///
+ /// This should only be used in situations where you really only want to
+ /// configure a specific restricted set of options.
+ factory SuiteConfiguration._unsafe(
+ {bool? allowTestRandomization,
+ bool? jsTrace,
+ bool? runSkipped,
+ Iterable<String>? dart2jsArgs,
+ String? precompiledPath,
+ Iterable<Pattern>? patterns,
+ Iterable<RuntimeSelection>? runtimes,
+ BooleanSelector? includeTags,
+ BooleanSelector? excludeTags,
+ Map<BooleanSelector, SuiteConfiguration>? tags,
+ Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+
+ // Test-level configuration
+ Timeout? timeout,
+ bool? verboseTrace,
+ bool? chainStackTraces,
+ bool? skip,
+ int? retry,
+ String? skipReason,
+ PlatformSelector? testOn,
+ Iterable<String>? addTags}) =>
+ SuiteConfiguration(
+ allowTestRandomization: allowTestRandomization,
+ jsTrace: jsTrace,
+ runSkipped: runSkipped,
+ dart2jsArgs: dart2jsArgs,
+ precompiledPath: precompiledPath,
+ patterns: patterns,
+ runtimes: runtimes,
+ includeTags: includeTags,
+ excludeTags: excludeTags,
+ tags: tags,
+ onPlatform: onPlatform,
+ timeout: timeout,
+ verboseTrace: verboseTrace,
+ chainStackTraces: chainStackTraces,
+ skip: skip,
+ retry: retry,
+ skipReason: skipReason,
+ testOn: testOn,
+ addTags: addTags);
+
+ /// A specialized constructor for only configuring the runtimes.
+ factory SuiteConfiguration.runtimes(Iterable<RuntimeSelection> runtimes) =>
+ SuiteConfiguration._unsafe(runtimes: runtimes);
+
+ /// A specialized constructor for only configuring runSkipped.
+ factory SuiteConfiguration.runSkipped(bool runSkipped) =>
+ SuiteConfiguration._unsafe(runSkipped: runSkipped);
+
+ /// A specialized constructor for only configuring the timeout.
+ factory SuiteConfiguration.timeout(Timeout timeout) =>
+ SuiteConfiguration._unsafe(timeout: timeout);
+
/// Creates new SuiteConfiguration.
///
/// Unlike [new SuiteConfiguration], this assumes [tags] is already
/// resolved.
SuiteConfiguration._(
- {bool? jsTrace,
- bool? runSkipped,
- Iterable<String>? dart2jsArgs,
- this.precompiledPath,
- Iterable<Pattern>? patterns,
- Iterable<RuntimeSelection>? runtimes,
- BooleanSelector? includeTags,
- BooleanSelector? excludeTags,
- Map<BooleanSelector, SuiteConfiguration>? tags,
- Map<PlatformSelector, SuiteConfiguration>? onPlatform,
- Metadata? metadata})
- : _jsTrace = jsTrace,
+ {required bool? allowTestRandomization,
+ required bool? jsTrace,
+ required bool? runSkipped,
+ required Iterable<String>? dart2jsArgs,
+ required this.precompiledPath,
+ required Iterable<Pattern>? patterns,
+ required Iterable<RuntimeSelection>? runtimes,
+ required BooleanSelector? includeTags,
+ required BooleanSelector? excludeTags,
+ required Map<BooleanSelector, SuiteConfiguration>? tags,
+ required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+ required Metadata? metadata})
+ : _allowTestRandomization = allowTestRandomization,
+ _jsTrace = jsTrace,
_runSkipped = runSkipped,
dart2jsArgs = _list(dart2jsArgs) ?? const [],
patterns = UnmodifiableSetView(patterns?.toSet() ?? {}),
@@ -199,11 +278,21 @@
/// [metadata].
factory SuiteConfiguration.fromMetadata(Metadata metadata) =>
SuiteConfiguration._(
- tags: metadata.forTag.map((key, child) =>
- MapEntry(key, SuiteConfiguration.fromMetadata(child))),
- onPlatform: metadata.onPlatform.map((key, child) =>
- MapEntry(key, SuiteConfiguration.fromMetadata(child))),
- metadata: metadata.change(forTag: {}, onPlatform: {}));
+ tags: metadata.forTag.map((key, child) =>
+ MapEntry(key, SuiteConfiguration.fromMetadata(child))),
+ onPlatform: metadata.onPlatform.map((key, child) =>
+ MapEntry(key, SuiteConfiguration.fromMetadata(child))),
+ metadata: metadata.change(forTag: {}, onPlatform: {}),
+ allowTestRandomization: null,
+ jsTrace: null,
+ runSkipped: null,
+ dart2jsArgs: null,
+ precompiledPath: null,
+ patterns: null,
+ runtimes: null,
+ includeTags: null,
+ excludeTags: null,
+ );
/// Returns an unmodifiable copy of [input].
///
@@ -231,6 +320,8 @@
if (other == SuiteConfiguration.empty) return this;
var config = SuiteConfiguration._(
+ allowTestRandomization:
+ other._allowTestRandomization ?? _allowTestRandomization,
jsTrace: other._jsTrace ?? _jsTrace,
runSkipped: other._runSkipped ?? _runSkipped,
dart2jsArgs: dart2jsArgs.toList()..addAll(other.dart2jsArgs),
@@ -250,7 +341,8 @@
/// Note that unlike [merge], this has no merging behavior—the old value is
/// always replaced by the new one.
SuiteConfiguration change(
- {bool? jsTrace,
+ {bool? allowTestRandomization,
+ bool? jsTrace,
bool? runSkipped,
Iterable<String>? dart2jsArgs,
String? precompiledPath,
@@ -271,6 +363,8 @@
PlatformSelector? testOn,
Iterable<String>? addTags}) {
var config = SuiteConfiguration._(
+ allowTestRandomization:
+ allowTestRandomization ?? _allowTestRandomization,
jsTrace: jsTrace ?? _jsTrace,
runSkipped: runSkipped ?? _runSkipped,
dart2jsArgs: dart2jsArgs?.toList() ?? this.dart2jsArgs,
diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml
index 24878e1..1b1b025 100644
--- a/pkgs/test_core/pubspec.yaml
+++ b/pkgs/test_core/pubspec.yaml
@@ -1,5 +1,5 @@
name: test_core
-version: 0.3.30-dev
+version: 0.4.0-dev
description: A basic library for writing tests and running them on the VM.
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_core
