Differentiate between test-randomize-ordering-seed=0 and null. (#1150)
* Differentiate between seed=0 and not set
* Add note to readme about shuffling tests
diff --git a/pkgs/test/README.md b/pkgs/test/README.md
index 7c91633..a661323 100644
--- a/pkgs/test/README.md
+++ b/pkgs/test/README.md
@@ -171,6 +171,19 @@
pub run test --total-shards 3 --shard-index 2 path/to/test.dart
```
+### Shuffling Tests
+Test order can be shuffled with the `--test-randomize-ordering-seed` argument.
+This allows you to shuffle your tests with a specific seed (deterministic) or
+a random seed for each run. For example, consider the following test runs:
+
+```bash
+pub run test --test-randomize-ordering-seed=12345
+pub run test --test-randomize-ordering-seed=random
+```
+
+Setting `--test-randomize-ordering-seed=0` will have the same effect as not
+specifying it at all, meaning the test order will remain as-is.
+
### Collecting Code Coverage
To collect code coverage, you can run tests with the `--coverage <directory>`
argument. The directory specified can be an absolute or relative path.
diff --git a/pkgs/test/test/runner/configuration/suite_test.dart b/pkgs/test/test/runner/configuration/suite_test.dart
index 56b8119..a1b4eb7 100644
--- a/pkgs/test/test/runner/configuration/suite_test.dart
+++ b/pkgs/test/test/runner/configuration/suite_test.dart
@@ -20,7 +20,7 @@
expect(merged.runSkipped, isFalse);
expect(merged.precompiledPath, isNull);
expect(merged.runtimes, equals([Runtime.vm.identifier]));
- expect(merged.testRandomizeOrderingSeed, 0);
+ expect(merged.testRandomizeOrderingSeed, null);
});
test("if only the old configuration's is defined, uses it", () {
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index e43f0ab..f28a372 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -3,6 +3,8 @@
* Bump minimum SDK to `2.4.0` for safer usage of for-loop elements.
* Deprecate `PhantomJS` and provide warning when used. Support for `PhantomJS`
will be removed in version `2.0.0`.
+* Differentiate between test-randomize-ordering-seed not set and 0 being chosen
+ as the random seed.
## 0.2.18
diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart
index ed7dd26..d4d8ab8 100644
--- a/pkgs/test_core/lib/src/runner/configuration/args.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/args.dart
@@ -220,7 +220,7 @@
var seed = value == 'random'
? Random().nextInt(4294967295)
: int.parse(value).toUnsigned(32);
- if (seed != null && seed > 0) {
+ if (seed != null) {
print('Shuffling test order with --test-randomize-ordering-seed=$seed');
}
return seed;
diff --git a/pkgs/test_core/lib/src/runner/engine.dart b/pkgs/test_core/lib/src/runner/engine.dart
index cf7dbd6..e4a7fa4 100644
--- a/pkgs/test_core/lib/src/runner/engine.dart
+++ b/pkgs/test_core/lib/src/runner/engine.dart
@@ -323,7 +323,8 @@
if (!_closed && setUpAllSucceeded) {
// shuffle the group entries
var entries = group.entries.toList();
- if (suiteConfig.testRandomizeOrderingSeed > 0) {
+ if (suiteConfig.testRandomizeOrderingSeed != null &&
+ suiteConfig.testRandomizeOrderingSeed > 0) {
entries.shuffle(Random(suiteConfig.testRandomizeOrderingSeed));
}
diff --git a/pkgs/test_core/lib/src/runner/suite.dart b/pkgs/test_core/lib/src/runner/suite.dart
index 233dc72..c95494a 100644
--- a/pkgs/test_core/lib/src/runner/suite.dart
+++ b/pkgs/test_core/lib/src/runner/suite.dart
@@ -85,10 +85,9 @@
final Map<PlatformSelector, SuiteConfiguration> onPlatform;
/// The seed with which to shuffle the test order.
- /// Default value is 0 if not provided and will not change the test order.
+ /// Default value is null if not provided and will not change the test order.
/// The same seed will shuffle the tests in the same way every time.
- int get testRandomizeOrderingSeed => _testRandomizeOrderingSeed ?? 0;
- final int _testRandomizeOrderingSeed;
+ final int testRandomizeOrderingSeed;
/// The global test metadata derived from this configuration.
Metadata get metadata {
@@ -202,7 +201,7 @@
excludeTags = excludeTags ?? BooleanSelector.none,
tags = _map(tags),
onPlatform = _map(onPlatform),
- _testRandomizeOrderingSeed = testRandomizeOrderingSeed,
+ testRandomizeOrderingSeed = testRandomizeOrderingSeed,
_metadata = metadata ?? Metadata.empty;
/// Creates a new [SuiteConfiguration] that takes its configuration from
@@ -252,7 +251,7 @@
tags: _mergeConfigMaps(tags, other.tags),
onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform),
testRandomizeOrderingSeed:
- other._testRandomizeOrderingSeed ?? _testRandomizeOrderingSeed,
+ other.testRandomizeOrderingSeed ?? testRandomizeOrderingSeed,
metadata: metadata.merge(other.metadata));
return config._resolveTags();
}
@@ -295,7 +294,7 @@
tags: tags ?? this.tags,
onPlatform: onPlatform ?? this.onPlatform,
testRandomizeOrderingSeed:
- testRandomizeOrderingSeed ?? _testRandomizeOrderingSeed,
+ testRandomizeOrderingSeed ?? testRandomizeOrderingSeed,
metadata: _metadata.change(
timeout: timeout,
verboseTrace: verboseTrace,
