Allow retry load suites (#1160)
Partial resolution for https://github.com/dart-lang/test/issues/1159.
Retries loading test suites (in loader.dart). Retry configuration is currently based on the suite metadata - and is counted separately from the test retries (does not count as a retry for tests).
diff --git a/pkgs/test/test/runner/loader_test.dart b/pkgs/test/test/runner/loader_test.dart
index 1311ece..9b2e7ac 100644
--- a/pkgs/test/test/runner/loader_test.dart
+++ b/pkgs/test/test/runner/loader_test.dart
@@ -3,7 +3,9 @@
// BSD-style license that can be found in the LICENSE file.
@TestOn('vm')
+import 'dart:async';
+import 'package:pedantic/pedantic.dart';
import 'package:path/path.dart' as p;
import 'package:test_descriptor/test_descriptor.dart' as d;
@@ -152,6 +154,48 @@
expectTestPassed(liveTest);
});
+ group('LoadException', () {
+ test('suites can be retried', () async {
+ var numRetries = 5;
+
+ await d.file('a_test.dart', '''
+ import 'hello.dart';
+
+ void main() {}
+ ''').create();
+
+ var firstFailureCompleter = Completer<void>();
+
+ // After the first load failure we create the missing dependency.
+ unawaited(firstFailureCompleter.future.then((_) async {
+ await d.file('hello.dart', '''
+ String get message => 'hello';
+ ''').create();
+ }));
+
+ await runZoned(() async {
+ var suites = await _loader
+ .loadFile(p.join(d.sandbox, 'a_test.dart'),
+ SuiteConfiguration(retry: numRetries))
+ .toList();
+ expect(suites, hasLength(1));
+ var loadSuite = suites.first;
+ var suite = await loadSuite.getSuite();
+ expect(suite.path, equals(p.join(d.sandbox, 'a_test.dart')));
+ expect(suite.platform.runtime, equals(Runtime.vm));
+ }, zoneSpecification:
+ ZoneSpecification(print: (_, parent, zone, message) {
+ if (message.contains('Retrying load of') &&
+ !firstFailureCompleter.isCompleted) {
+ firstFailureCompleter.complete(null);
+ }
+ parent.print(zone, message);
+ }));
+
+ expect(firstFailureCompleter.isCompleted, true);
+ });
+ });
+
// TODO: Test load suites. Don't forget to test that prints in loaded files
// are piped through the suite. Also for browser tests!
}
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index f16d824..b36455a 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -6,6 +6,9 @@
* Differentiate between test-randomize-ordering-seed not set and 0 being chosen
as the random seed.
* `deserializeSuite` now takes an optional `gatherCoverage` callback.
+* Support retrying of entire test suites when they fail to load.
+* Fix the `compiling` message in precompiled mode so it says `loading` instead,
+ which is more accurate.
## 0.2.18
diff --git a/pkgs/test_core/lib/src/runner/loader.dart b/pkgs/test_core/lib/src/runner/loader.dart
index cdc64c8..b8c779f 100644
--- a/pkgs/test_core/lib/src/runner/loader.dart
+++ b/pkgs/test_core/lib/src/runner/loader.dart
@@ -215,21 +215,37 @@
continue;
}
- var name = (platform.runtime.isJS ? 'compiling ' : 'loading ') + path;
+ var name =
+ (platform.runtime.isJS && platformConfig.precompiledPath == null
+ ? 'compiling '
+ : 'loading ') +
+ path;
yield LoadSuite(name, platformConfig, platform, () async {
var memo = _platformPlugins[platform.runtime];
- try {
- var plugin = await memo.runOnce(_platformCallbacks[platform.runtime]);
- _customizePlatform(plugin, platform.runtime);
- var suite = await plugin.load(path, platform, platformConfig,
- {'platformVariables': _runtimeVariables.toList()});
- if (suite != null) _suites.add(suite);
- return suite;
- } catch (error, stackTrace) {
- if (error is LoadException) rethrow;
- await Future.error(LoadException(path, error), stackTrace);
- return null;
+ var retriesLeft = suiteConfig.metadata.retry;
+ while (true) {
+ try {
+ var plugin =
+ await memo.runOnce(_platformCallbacks[platform.runtime]);
+ _customizePlatform(plugin, platform.runtime);
+ var suite = await plugin.load(path, platform, platformConfig,
+ {'platformVariables': _runtimeVariables.toList()});
+ if (suite != null) _suites.add(suite);
+ return suite;
+ } catch (error, stackTrace) {
+ if (retriesLeft > 0) {
+ retriesLeft--;
+ print('Retrying load of $path in 1s ($retriesLeft remaining)');
+ await Future.delayed(Duration(seconds: 1));
+ continue;
+ }
+ if (error is LoadException) {
+ rethrow;
+ }
+ await Future.error(LoadException(path, error), stackTrace);
+ return null;
+ }
}
}, path: path);
}
