add --ignore-timeouts flag (#1635)
Closes https://github.com/dart-lang/test/issues/1634
Adds an `ignoreTimeouts` field on `Suite`, which is read through the invoker to ignore its timeout behavior.
I also changed the `--pause-after-load` behavior slightly so that it implies this new flag instead of `--timeout=none`. This flag more reliably achieves the desired result (not timeouts) as it won't be overridden by local test config.
diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md
index cc7d36f..508164b 100644
--- a/pkgs/test/CHANGELOG.md
+++ b/pkgs/test/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 1.20.0-dev
+
+* Add an `--ignore-timeouts` command line flag, which disables all timeouts
+ for all tests. This can be useful when debugging, so tests don't time out
+ during debug sessions.
+
## 1.19.4
* Wait for paused VM platform isolates before shutdown.
diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml
index 281af65..3ca7129 100644
--- a/pkgs/test/pubspec.yaml
+++ b/pkgs/test/pubspec.yaml
@@ -1,5 +1,5 @@
name: test
-version: 1.19.4
+version: 1.20.0-dev
description: >-
A full featured library for writing and running Dart tests across platforms.
repository: https://github.com/dart-lang/test/blob/master/pkgs/test
@@ -32,8 +32,8 @@
webkit_inspection_protocol: ^1.0.0
yaml: ^3.0.0
# Use an exact version until the test_api and test_core package are stable.
- test_api: 0.4.8
- test_core: 0.4.9
+ test_api: 0.4.9
+ test_core: 0.4.10
dev_dependencies:
fake_async: ^1.0.0
diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart
index 7ebc824..c393883 100644
--- a/pkgs/test/test/runner/runner_test.dart
+++ b/pkgs/test/test/runner/runner_test.dart
@@ -82,8 +82,9 @@
--pub-serve=<port> The port of a pub serve instance serving "test/".
--timeout The default test timeout. For example: 15s, 2x, none
(defaults to "30s")
+ --ignore-timeouts Ignore all timeouts (useful if debugging)
--pause-after-load Pause for debugging before any tests execute.
- Implies --concurrency=1, --debug, and --timeout=none.
+ Implies --concurrency=1, --debug, and --ignore-timeouts.
Currently only supported for browser tests.
--debug Run the VM and Chrome tests in debug mode.
--coverage=<directory> Gather coverage and output it to the specified directory.
diff --git a/pkgs/test/test/runner/timeout_test.dart b/pkgs/test/test/runner/timeout_test.dart
index 598b95d..b5d3242 100644
--- a/pkgs/test/test/runner/timeout_test.dart
+++ b/pkgs/test/test/runner/timeout_test.dart
@@ -162,4 +162,24 @@
['Test timed out after 0 seconds.', '-1: Some tests failed.']));
await test.shouldExit(1);
});
+
+ test('are ignored with --ignore-timeouts', () async {
+ await d.file('test.dart', '''
+@Timeout(const Duration(seconds: 0))
+
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+ test("timeout", () async {
+ await Future.delayed(Duration(milliseconds: 10));
+ });
+}
+''').create();
+
+ var test = await runTest(['test.dart', '--ignore-timeouts']);
+ expect(test.stdout, containsInOrder(['+1: All tests passed!']));
+ await test.shouldExit(0);
+ });
}
diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart
index e14fa51..f62eba6 100644
--- a/pkgs/test/test/utils.dart
+++ b/pkgs/test/test/utils.dart
@@ -109,7 +109,7 @@
/// Returns a local [LiveTest] that runs [body].
LiveTest createTest(dynamic Function() body) {
var test = LocalTest('test', Metadata(chainStackTraces: true), body);
- var suite = Suite(Group.root([test]), suitePlatform);
+ var suite = Suite(Group.root([test]), suitePlatform, ignoreTimeouts: false);
return test.load(suite);
}
@@ -289,6 +289,7 @@
Map<String, CustomRuntime>? defineRuntimes,
bool? noRetry,
bool? useDataIsolateStrategy,
+ bool? ignoreTimeouts,
// Suite-level configuration
bool? allowDuplicateTestNames,
@@ -340,6 +341,7 @@
defineRuntimes: defineRuntimes,
noRetry: noRetry,
useDataIsolateStrategy: useDataIsolateStrategy,
+ ignoreTimeouts: ignoreTimeouts,
allowDuplicateTestNames: allowDuplicateTestNames,
allowTestRandomization: allowTestRandomization,
jsTrace: jsTrace,
diff --git a/pkgs/test_api/CHANGELOG.md b/pkgs/test_api/CHANGELOG.md
index 5cef5ce..5872976 100644
--- a/pkgs/test_api/CHANGELOG.md
+++ b/pkgs/test_api/CHANGELOG.md
@@ -1,3 +1,8 @@
+## 0.4.9-dev
+
+* Add `ignoreTimeouts` option to `Suite`, which disables all timeouts for all
+ tests in that suite.
+
## 0.4.8
* `TestFailure` implements `Exception` for compatibility with
diff --git a/pkgs/test_api/lib/src/backend/invoker.dart b/pkgs/test_api/lib/src/backend/invoker.dart
index dfc67fe..1ce55c9 100644
--- a/pkgs/test_api/lib/src/backend/invoker.dart
+++ b/pkgs/test_api/lib/src/backend/invoker.dart
@@ -270,6 +270,7 @@
void heartbeat() {
if (liveTest.isComplete) return;
if (_timeoutTimer != null) _timeoutTimer!.cancel();
+ if (liveTest.suite.ignoreTimeouts == true) return;
const defaultTimeout = Duration(seconds: 30);
var timeout = liveTest.test.metadata.timeout.apply(defaultTimeout);
diff --git a/pkgs/test_api/lib/src/backend/remote_listener.dart b/pkgs/test_api/lib/src/backend/remote_listener.dart
index 65a4ca5..00c78e1 100644
--- a/pkgs/test_api/lib/src/backend/remote_listener.dart
+++ b/pkgs/test_api/lib/src/backend/remote_listener.dart
@@ -119,9 +119,12 @@
await declarer.declare(main);
- var suite = Suite(declarer.build(),
- SuitePlatform.deserialize(message['platform'] as Object),
- path: message['path'] as String);
+ var suite = Suite(
+ declarer.build(),
+ SuitePlatform.deserialize(message['platform'] as Object),
+ path: message['path'] as String,
+ ignoreTimeouts: message['ignoreTimeouts'] as bool? ?? false,
+ );
runZoned(() {
Invoker.guard(
diff --git a/pkgs/test_api/lib/src/backend/suite.dart b/pkgs/test_api/lib/src/backend/suite.dart
index 9321352..08719bd 100644
--- a/pkgs/test_api/lib/src/backend/suite.dart
+++ b/pkgs/test_api/lib/src/backend/suite.dart
@@ -26,13 +26,16 @@
/// The top-level group for this test suite.
final Group group;
+ /// Whether or not to ignore test timeouts.
+ final bool ignoreTimeouts;
+
/// Creates a new suite containing [entires].
///
/// If [platform] and/or [os] are passed, [group] is filtered to match that
/// platform information.
///
/// If [os] is passed without [platform], throws an [ArgumentError].
- Suite(Group group, this.platform, {this.path})
+ Suite(Group group, this.platform, {required this.ignoreTimeouts, this.path})
: group = _filterGroup(group, platform);
/// Returns [entries] filtered according to [platform] and [os].
@@ -51,7 +54,8 @@
Suite filter(bool Function(Test) callback) {
var filtered = group.filter(callback);
filtered ??= Group.root([], metadata: metadata);
- return Suite(filtered, platform, path: path);
+ return Suite(filtered, platform,
+ ignoreTimeouts: ignoreTimeouts, path: path);
}
bool get isLoadSuite => false;
diff --git a/pkgs/test_api/pubspec.yaml b/pkgs/test_api/pubspec.yaml
index 07e9206..fcda266 100644
--- a/pkgs/test_api/pubspec.yaml
+++ b/pkgs/test_api/pubspec.yaml
@@ -1,5 +1,5 @@
name: test_api
-version: 0.4.8
+version: 0.4.9-dev
description: A library for writing Dart tests.
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_api
diff --git a/pkgs/test_api/test/backend/declarer_test.dart b/pkgs/test_api/test/backend/declarer_test.dart
index 0b709cc..e8bf0a9 100644
--- a/pkgs/test_api/test/backend/declarer_test.dart
+++ b/pkgs/test_api/test/backend/declarer_test.dart
@@ -17,7 +17,7 @@
void main() {
setUp(() {
- _suite = Suite(Group.root([]), suitePlatform);
+ _suite = Suite(Group.root([]), suitePlatform, ignoreTimeouts: false);
});
group('.test()', () {
diff --git a/pkgs/test_api/test/backend/invoker_test.dart b/pkgs/test_api/test/backend/invoker_test.dart
index d30c250..f430c0b 100644
--- a/pkgs/test_api/test/backend/invoker_test.dart
+++ b/pkgs/test_api/test/backend/invoker_test.dart
@@ -19,7 +19,7 @@
late Suite suite;
setUp(() {
lastState = null;
- suite = Suite(Group.root([]), suitePlatform);
+ suite = Suite(Group.root([]), suitePlatform, ignoreTimeouts: false);
});
group('Invoker.current', () {
@@ -428,8 +428,7 @@
Invoker.current!.addOutstandingCallback();
},
metadata: Metadata(
- chainStackTraces: true,
- timeout: Timeout(Duration(milliseconds: 100))))
+ chainStackTraces: true, timeout: Timeout(Duration.zero)))
.load(suite);
expectStates(liveTest, [
@@ -446,6 +445,23 @@
liveTest.run();
});
+
+ test('can be ignored', () {
+ suite = Suite(Group.root([]), suitePlatform, ignoreTimeouts: true);
+ var liveTest = _localTest(() async {
+ await Future.delayed(Duration(milliseconds: 10));
+ },
+ metadata: Metadata(
+ chainStackTraces: true, timeout: Timeout(Duration.zero)))
+ .load(suite);
+
+ expectStates(liveTest, [
+ const State(Status.running, Result.success),
+ const State(Status.complete, Result.success)
+ ]);
+
+ liveTest.run();
+ });
});
group('runTearDowns', () {
diff --git a/pkgs/test_api/test/utils.dart b/pkgs/test_api/test/utils.dart
index a9fc684..04c37f2 100644
--- a/pkgs/test_api/test/utils.dart
+++ b/pkgs/test_api/test/utils.dart
@@ -91,7 +91,7 @@
/// Returns a local [LiveTest] that runs [body].
LiveTest createTest(dynamic Function() body) {
var test = LocalTest('test', Metadata(chainStackTraces: true), body);
- var suite = Suite(Group.root([test]), suitePlatform);
+ var suite = Suite(Group.root([test]), suitePlatform, ignoreTimeouts: false);
return test.load(suite);
}
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index da0065c..65db89a 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -1,3 +1,8 @@
+## 0.4.10
+
+* Add an `--ignore-timeouts` command line flag, which disables all timeouts
+ for all tests.
+
## 0.4.9
* Wait for paused VM platform isolates before shutdown.
diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart
index 6286db6..8d78df7 100644
--- a/pkgs/test_core/lib/src/runner/configuration.dart
+++ b/pkgs/test_core/lib/src/runner/configuration.dart
@@ -220,6 +220,9 @@
/// The same seed will shuffle the tests in the same way every time.
final int? testRandomizeOrderingSeed;
+ final bool? _ignoreTimeouts;
+ bool get ignoreTimeouts => _ignoreTimeouts ?? false;
+
/// Returns the current configuration, or a default configuration if no
/// current configuration is set.
///
@@ -285,6 +288,7 @@
required bool? noRetry,
required bool? useDataIsolateStrategy,
required int? testRandomizeOrderingSeed,
+ required bool? ignoreTimeouts,
// Suite-level configuration
required bool? allowDuplicateTestNames,
@@ -337,6 +341,7 @@
noRetry: noRetry,
useDataIsolateStrategy: useDataIsolateStrategy,
testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ ignoreTimeouts: ignoreTimeouts,
suiteDefaults: SuiteConfiguration(
allowDuplicateTestNames: allowDuplicateTestNames,
allowTestRandomization: allowTestRandomization,
@@ -395,9 +400,11 @@
Map<String, CustomRuntime>? defineRuntimes,
bool? noRetry,
bool? useDataIsolateStrategy,
- bool? allowDuplicateTestNames,
+ int? testRandomizeOrderingSeed,
+ bool? ignoreTimeouts,
// Suite-level configuration
+ bool? allowDuplicateTestNames,
bool? allowTestRandomization,
bool? jsTrace,
bool? runSkipped,
@@ -409,7 +416,6 @@
BooleanSelector? excludeTags,
Map<BooleanSelector, SuiteConfiguration>? tags,
Map<PlatformSelector, SuiteConfiguration>? onPlatform,
- int? testRandomizeOrderingSeed,
// Test-level configuration
Timeout? timeout,
@@ -446,6 +452,8 @@
defineRuntimes: defineRuntimes,
noRetry: noRetry,
useDataIsolateStrategy: useDataIsolateStrategy,
+ testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ ignoreTimeouts: ignoreTimeouts,
allowDuplicateTestNames: allowDuplicateTestNames,
allowTestRandomization: allowTestRandomization,
jsTrace: jsTrace,
@@ -458,7 +466,6 @@
excludeTags: excludeTags,
tags: tags,
onPlatform: onPlatform,
- testRandomizeOrderingSeed: testRandomizeOrderingSeed,
timeout: timeout,
verboseTrace: verboseTrace,
chainStackTraces: chainStackTraces,
@@ -513,6 +520,7 @@
noRetry: null,
useDataIsolateStrategy: null,
testRandomizeOrderingSeed: null,
+ ignoreTimeouts: null,
allowDuplicateTestNames: null,
allowTestRandomization: null,
runSkipped: null,
@@ -580,6 +588,7 @@
noRetry: null,
useDataIsolateStrategy: null,
testRandomizeOrderingSeed: null,
+ ignoreTimeouts: null,
jsTrace: null,
runSkipped: null,
dart2jsArgs: null,
@@ -642,6 +651,7 @@
noRetry: null,
useDataIsolateStrategy: null,
testRandomizeOrderingSeed: null,
+ ignoreTimeouts: null,
allowDuplicateTestNames: null,
allowTestRandomization: null,
jsTrace: null,
@@ -705,6 +715,7 @@
noRetry: null,
useDataIsolateStrategy: null,
testRandomizeOrderingSeed: null,
+ ignoreTimeouts: null,
allowDuplicateTestNames: null,
allowTestRandomization: null,
jsTrace: null,
@@ -770,6 +781,7 @@
required bool? noRetry,
required bool? useDataIsolateStrategy,
required this.testRandomizeOrderingSeed,
+ required bool? ignoreTimeouts,
required SuiteConfiguration? suiteDefaults})
: _help = help,
_version = version,
@@ -794,10 +806,9 @@
defineRuntimes = _map(defineRuntimes),
_noRetry = noRetry,
_useDataIsolateStrategy = useDataIsolateStrategy,
- suiteDefaults = pauseAfterLoad == true
- ? suiteDefaults?.change(timeout: Timeout.none) ??
- SuiteConfiguration.timeout(Timeout.none)
- : suiteDefaults ?? SuiteConfiguration.empty {
+ _ignoreTimeouts =
+ pauseAfterLoad == true ? ignoreTimeouts ?? true : ignoreTimeouts,
+ suiteDefaults = suiteDefaults ?? SuiteConfiguration.empty {
if (_filename != null && _filename!.context.style != p.style) {
throw ArgumentError(
"filename's context must match the current operating system, was "
@@ -845,6 +856,7 @@
noRetry: null,
useDataIsolateStrategy: null,
testRandomizeOrderingSeed: null,
+ ignoreTimeouts: null,
);
/// Returns an unmodifiable copy of [input].
@@ -961,6 +973,7 @@
other._useDataIsolateStrategy ?? _useDataIsolateStrategy,
testRandomizeOrderingSeed:
other.testRandomizeOrderingSeed ?? testRandomizeOrderingSeed,
+ ignoreTimeouts: other._ignoreTimeouts ?? ignoreTimeouts,
suiteDefaults: suiteDefaults.merge(other.suiteDefaults));
result = result._resolvePresets();
@@ -1000,6 +1013,8 @@
Map<String, CustomRuntime>? defineRuntimes,
bool? noRetry,
bool? useDataIsolateStrategy,
+ int? testRandomizeOrderingSeed,
+ bool? ignoreTimeouts,
// Suite-level configuration
bool? allowDuplicateTestNames,
@@ -1013,7 +1028,6 @@
BooleanSelector? excludeTags,
Map<BooleanSelector, SuiteConfiguration>? tags,
Map<PlatformSelector, SuiteConfiguration>? onPlatform,
- int? testRandomizeOrderingSeed,
// Test-level configuration
Timeout? timeout,
@@ -1053,6 +1067,7 @@
useDataIsolateStrategy ?? _useDataIsolateStrategy,
testRandomizeOrderingSeed:
testRandomizeOrderingSeed ?? this.testRandomizeOrderingSeed,
+ ignoreTimeouts: ignoreTimeouts ?? _ignoreTimeouts,
suiteDefaults: suiteDefaults.change(
allowDuplicateTestNames: allowDuplicateTestNames,
jsTrace: jsTrace,
diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart
index ed2ef40..baed5fd 100644
--- a/pkgs/test_core/lib/src/runner/configuration/args.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/args.dart
@@ -88,9 +88,11 @@
parser.addOption('timeout',
help: 'The default test timeout. For example: 15s, 2x, none',
defaultsTo: '30s');
+ parser.addFlag('ignore-timeouts',
+ help: 'Ignore all timeouts (useful if debugging)', negatable: false);
parser.addFlag('pause-after-load',
help: 'Pause for debugging before any tests execute.\n'
- 'Implies --concurrency=1, --debug, and --timeout=none.\n'
+ 'Implies --concurrency=1, --debug, and --ignore-timeouts.\n'
'Currently only supported for browser tests.',
negatable: false);
parser.addFlag('debug',
@@ -312,6 +314,7 @@
noRetry: _ifParsed('no-retry'),
useDataIsolateStrategy: _ifParsed('use-data-isolate-strategy'),
testRandomizeOrderingSeed: testRandomizeOrderingSeed,
+ ignoreTimeouts: _ifParsed('ignore-timeouts'),
// Config that isn't supported on the command line
addTags: null,
allowTestRandomization: null,
diff --git a/pkgs/test_core/lib/src/runner/load_suite.dart b/pkgs/test_core/lib/src/runner/load_suite.dart
index 92640bb..cb17a78 100644
--- a/pkgs/test_core/lib/src/runner/load_suite.dart
+++ b/pkgs/test_core/lib/src/runner/load_suite.dart
@@ -18,6 +18,7 @@
import '../util/async.dart';
import '../util/io_stub.dart' if (dart.library.io) '../util/io.dart';
import '../util/pair.dart';
+import 'configuration.dart';
import 'load_exception.dart';
import 'plugin/environment.dart';
import 'runner_suite.dart';
@@ -116,7 +117,8 @@
// If the test is forcibly closed, let it complete, since load tests don't
// have timeouts.
invoker.onClose.then((_) => invoker.removeOutstandingCallback());
- }, completer.future, path: path);
+ }, completer.future,
+ path: path, ignoreTimeouts: Configuration.current.ignoreTimeouts);
}
/// A utility constructor for a load suite that just throws [exception].
@@ -143,23 +145,27 @@
}
LoadSuite._(String name, this.config, SuitePlatform platform,
- void Function() body, this._suiteAndZone, {String? path})
+ void Function() body, this._suiteAndZone,
+ {required bool ignoreTimeouts, String? path})
: super(
Group.root(
[LocalTest(name, Metadata(timeout: Timeout(_timeout)), body)]),
platform,
- path: path);
+ path: path,
+ ignoreTimeouts: ignoreTimeouts);
/// A constructor used by [changeSuite].
LoadSuite._changeSuite(LoadSuite old, this._suiteAndZone)
: config = old.config,
- super(old.group, old.platform, path: old.path);
+ super(old.group, old.platform,
+ path: old.path, ignoreTimeouts: old.ignoreTimeouts);
/// A constructor used by [filter].
LoadSuite._filtered(LoadSuite old, Group filtered)
: config = old.config,
_suiteAndZone = old._suiteAndZone,
- super(old.group, old.platform, path: old.path);
+ super(old.group, old.platform,
+ path: old.path, ignoreTimeouts: old.ignoreTimeouts);
/// Creates a new [LoadSuite] that's identical to this one, but that
/// transforms [suite] once it's loaded.
diff --git a/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart b/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
index 2c9ea13..78132ff 100644
--- a/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
+++ b/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
@@ -62,6 +62,7 @@
'foldTraceExcept': Configuration.current.foldTraceExcept.toList(),
'foldTraceOnly': Configuration.current.foldTraceOnly.toList(),
'allowDuplicateTestNames': suiteConfig.allowDuplicateTestNames,
+ 'ignoreTimeouts': Configuration.current.ignoreTimeouts,
...(message as Map<String, dynamic>),
});
diff --git a/pkgs/test_core/lib/src/runner/runner_suite.dart b/pkgs/test_core/lib/src/runner/runner_suite.dart
index 9e9b917..ffdc51a 100644
--- a/pkgs/test_core/lib/src/runner/runner_suite.dart
+++ b/pkgs/test_core/lib/src/runner/runner_suite.dart
@@ -11,6 +11,7 @@
import 'package:test_api/src/backend/suite_platform.dart'; // ignore: implementation_imports
import 'package:test_api/src/backend/test.dart'; // ignore: implementation_imports
+import 'configuration.dart';
import 'environment.dart';
import 'suite.dart';
@@ -58,7 +59,8 @@
RunnerSuite._(this._controller, Group group, SuitePlatform platform,
{String? path})
- : super(group, platform, path: path);
+ : super(group, platform,
+ path: path, ignoreTimeouts: Configuration.current.ignoreTimeouts);
@override
RunnerSuite filter(bool Function(Test) callback) {
diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml
index 68a6442..17c9f9d 100644
--- a/pkgs/test_core/pubspec.yaml
+++ b/pkgs/test_core/pubspec.yaml
@@ -1,5 +1,5 @@
name: test_core
-version: 0.4.9
+version: 0.4.10-dev
description: A basic library for writing tests and running them on the VM.
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_core
@@ -30,7 +30,7 @@
# matcher is tightly constrained by test_api
matcher: any
# Use an exact version until the test_api package is stable.
- test_api: 0.4.8
+ test_api: 0.4.9
dev_dependencies:
lints: ^1.0.0