Add support for a timeout flag.
Closes #279
R=lrn@google.com, kevmoo@google.com
Review URL: https://codereview.chromium.org//1604043003 .
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8f7c789..3413bc2 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -2,6 +2,8 @@
* Organize the `--help` output into sections.
+* Add a `--timeout` flag.
+
## 0.12.7
* Add the ability to re-run tests while debugging. When the browser is paused at
diff --git a/lib/src/frontend/timeout.dart b/lib/src/frontend/timeout.dart
index 54ce540..9aca1c0 100644
--- a/lib/src/frontend/timeout.dart
+++ b/lib/src/frontend/timeout.dart
@@ -2,6 +2,21 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
+import 'package:string_scanner/string_scanner.dart';
+
+/// A regular expression that matches text until a letter or whitespace.
+///
+/// This is intended to scan through a number without actually encoding the full
+/// Dart number grammar. It doesn't stop on "e" because that can be a component
+/// of numbers.
+final _untilUnit = new RegExp(r"[^a-df-z\s]+", caseSensitive: false);
+
+/// A regular expression that matches a time unit.
+final _unit = new RegExp(r"([um]s|[dhms])", caseSensitive: false);
+
+/// A regular expression that matches a section of whitespace.
+final _whitespace = new RegExp(r"\s+");
+
/// A class representing a modification to the default timeout for a test.
///
/// By default, a test will time out after 30 seconds. With [new Timeout], that
@@ -39,10 +54,76 @@
: scaleFactor = null,
duration = null;
+ /// Parse the timeout from a user-provided string.
+ ///
+ /// This supports the following formats:
+ ///
+ /// * `Number "x"`, which produces a relative timeout with the given scale
+ /// factor.
+ ///
+ /// * `(Number ("d" | "h" | "m" | "s" | "ms" | "us") (" ")?)+`, which produces
+ /// an absolute timeout with the duration given by the sum of the given
+ /// units.
+ ///
+ /// * `"none"`, which produces [Timeout.none].
+ ///
+ /// Throws a [FormatException] if [timeout] is not in a valid format
+ factory Timeout.parse(String timeout) {
+ var scanner = new StringScanner(timeout);
+
+ // First check for the string "none".
+ if (scanner.scan("none")) {
+ scanner.expectDone();
+ return Timeout.none;
+ }
+
+ // Scan a number. This will be either a time unit or a scale factor.
+ scanner.expect(_untilUnit, name: "number");
+ var number = double.parse(scanner.lastMatch[0]);
+
+ // A number followed by "x" is a scale factor.
+ if (scanner.scan("x") || scanner.scan("X")) {
+ scanner.expectDone();
+ return new Timeout.factor(number);
+ }
+
+ // Parse time units until none are left. The condition is in the middle of
+ // the loop because we've already parsed the first number.
+ var microseconds = 0;
+ while (true) {
+ scanner.expect(_unit, name: "unit");
+ microseconds += _microsecondsFor(number, scanner.lastMatch[0]);
+
+ scanner.scan(_whitespace);
+
+ // Scan the next number, if it's avaialble.
+ if (!scanner.scan(_untilUnit)) break;
+ number = double.parse(scanner.lastMatch[0]);
+ }
+
+ scanner.expectDone();
+ return new Timeout(new Duration(microseconds: microseconds.round()));
+ }
+
+ /// Returns the number of microseconds in [number] [unit]s.
+ static double _microsecondsFor(double number, String unit) {
+ switch (unit) {
+ case "d": return number * 24 * 60 * 60 * 1000000;
+ case "h": return number * 60 * 60 * 1000000;
+ case "m": return number * 60 * 1000000;
+ case "s": return number * 1000000;
+ case "ms": return number * 1000;
+ case "us": return number;
+ default: throw new ArgumentError("Unknown unit $unit.");
+ }
+ }
+
/// Returns a new [Timeout] that merges [this] with [other].
///
- /// If [other] declares a [duration], that takes precedence. Otherwise, this
- /// timeout's [duration] or [factor] are multiplied by [other]'s [factor].
+ /// [Timeout.none] takes precedence over everything. If timeout is
+ /// [Timeout.none] and [other] declares a [duration], that takes precedence.
+ /// Otherwise, this timeout's [duration] or [factor] are multiplied by
+ /// [other]'s [factor].
Timeout merge(Timeout other) {
if (this == none || other == none) return none;
if (other.duration != null) return new Timeout(other.duration);
@@ -58,6 +139,11 @@
return duration == null ? base * scaleFactor : duration;
}
+ int get hashCode => duration.hashCode ^ 5 * scaleFactor.hashCode;
+
+ bool operator==(other) => other is Timeout && other.duration == duration &&
+ other.scaleFactor == scaleFactor;
+
String toString() {
if (duration != null) return duration.toString();
if (scaleFactor != null) return "${scaleFactor}x";
diff --git a/lib/src/runner/configuration.dart b/lib/src/runner/configuration.dart
index 2876f15..b9e8494 100644
--- a/lib/src/runner/configuration.dart
+++ b/lib/src/runner/configuration.dart
@@ -72,9 +72,16 @@
parser.addOption("pub-serve",
help: 'The port of a pub serve instance serving "test/".',
valueHelp: 'port');
+
+ // Note: although we list the 30s default timeout as though it were a
+ // default value for this argument, it's actually encoded in the [Invoker]'s
+ // call to [Timeout.apply].
+ parser.addOption("timeout",
+ help: 'The default test timeout. For example: 15s, 2x, none\n'
+ '(defaults to 30s)');
parser.addFlag("pause-after-load",
help: 'Pauses for debugging before any tests execute.\n'
- 'Implies --concurrency=1.\n'
+ 'Implies --concurrency=1 and --timeout=none.\n'
'Currently only supported for browser tests.',
negatable: false);
@@ -129,6 +136,9 @@
/// if tests should be loaded from the filesystem.
final Uri pubServeUrl;
+ /// The default test timeout.
+ final Timeout timeout;
+
/// Whether to use command-line color escapes.
final bool color;
@@ -156,9 +166,7 @@
/// The global test metadata derived from this configuration.
Metadata get metadata =>
- new Metadata(
- timeout: pauseAfterLoad ? Timeout.none : null,
- verboseTrace: verboseTrace);
+ new Metadata(timeout: timeout, verboseTrace: verboseTrace);
/// Parses the configuration from [args].
///
@@ -208,6 +216,9 @@
pubServePort: _wrapFormatException(options, 'pub-serve', int.parse),
concurrency: _wrapFormatException(options, 'concurrency', int.parse,
orElse: () => _defaultConcurrency),
+ timeout: _wrapFormatException(options, 'timeout',
+ (value) => new Timeout.parse(value),
+ orElse: () => new Timeout.factor(1)),
pattern: pattern,
platforms: options['platform'].map(TestPlatform.find),
paths: options.rest.isEmpty ? null : options.rest,
@@ -233,9 +244,9 @@
Configuration({this.help: false, this.version: false,
this.verboseTrace: false, this.jsTrace: false,
bool pauseAfterLoad: false, bool color, String packageRoot,
- String reporter, int pubServePort, int concurrency, this.pattern,
- Iterable<TestPlatform> platforms, Iterable<String> paths,
- Set<String> tags, Set<String> excludeTags})
+ String reporter, int pubServePort, int concurrency, Timeout timeout,
+ this.pattern, Iterable<TestPlatform> platforms,
+ Iterable<String> paths, Set<String> tags, Set<String> excludeTags})
: pauseAfterLoad = pauseAfterLoad,
color = color == null ? canUseSpecialChars : color,
packageRoot = packageRoot == null
@@ -248,6 +259,9 @@
concurrency = pauseAfterLoad
? 1
: (concurrency == null ? _defaultConcurrency : concurrency),
+ timeout = pauseAfterLoad
+ ? Timeout.none
+ : (timeout == null ? new Timeout.factor(1) : timeout),
platforms = platforms == null ? [TestPlatform.vm] : platforms.toList(),
paths = paths == null ? ["test"] : paths.toList(),
explicitPaths = paths != null,
diff --git a/pubspec.yaml b/pubspec.yaml
index 8d6472e..71f545c 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -1,5 +1,5 @@
name: test
-version: 0.12.8-dev
+version: 0.12.8
author: Dart Team <misc@dartlang.org>
description: A library for writing dart unit tests.
homepage: https://github.com/dart-lang/test
diff --git a/test/frontend/timeout_test.dart b/test/frontend/timeout_test.dart
new file mode 100644
index 0000000..382466c
--- /dev/null
+++ b/test/frontend/timeout_test.dart
@@ -0,0 +1,85 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:test/src/backend/state.dart';
+import 'package:test/test.dart';
+
+void main() {
+ group("Timeout.parse", () {
+ group('for "none"', () {
+ test("successfully parses", () {
+ expect(new Timeout.parse("none"), equals(Timeout.none));
+ });
+
+ test("rejects invalid input", () {
+ expect(() => new Timeout.parse(" none"), throwsFormatException);
+ expect(() => new Timeout.parse("none "), throwsFormatException);
+ expect(() => new Timeout.parse("xnone"), throwsFormatException);
+ expect(() => new Timeout.parse("nonex"), throwsFormatException);
+ expect(() => new Timeout.parse("noxe"), throwsFormatException);
+ });
+ });
+
+ group("for a relative timeout", () {
+ test("successfully parses", () {
+ expect(new Timeout.parse("1x"), equals(new Timeout.factor(1)));
+ expect(new Timeout.parse("2.5x"), equals(new Timeout.factor(2.5)));
+ expect(new Timeout.parse("1.2e3x"), equals(new Timeout.factor(1.2e3)));
+ });
+
+ test("rejects invalid input", () {
+ expect(() => new Timeout.parse(".x"), throwsFormatException);
+ expect(() => new Timeout.parse("x"), throwsFormatException);
+ expect(() => new Timeout.parse("ax"), throwsFormatException);
+ expect(() => new Timeout.parse("1x "), throwsFormatException);
+ expect(() => new Timeout.parse("1x5m"), throwsFormatException);
+ });
+ });
+
+ group("for an absolute timeout", () {
+ test("successfully parses all supported units", () {
+ expect(new Timeout.parse("2d"),
+ equals(new Timeout(new Duration(days: 2))));
+ expect(new Timeout.parse("2h"),
+ equals(new Timeout(new Duration(hours: 2))));
+ expect(new Timeout.parse("2m"),
+ equals(new Timeout(new Duration(minutes: 2))));
+ expect(new Timeout.parse("2s"),
+ equals(new Timeout(new Duration(seconds: 2))));
+ expect(new Timeout.parse("2ms"),
+ equals(new Timeout(new Duration(milliseconds: 2))));
+ expect(new Timeout.parse("2us"),
+ equals(new Timeout(new Duration(microseconds: 2))));
+ });
+
+ test("supports non-integer units", () {
+ expect(new Timeout.parse("2.73d"),
+ equals(new Timeout(new Duration(days: 1) * 2.73)));
+ });
+
+ test("supports multiple units", () {
+ expect(new Timeout.parse("1d 2h3m 4s5ms\t6us"),
+ equals(new Timeout(new Duration(
+ days: 1,
+ hours: 2,
+ minutes: 3,
+ seconds: 4,
+ milliseconds: 5,
+ microseconds: 6))));
+ });
+
+ test("rejects invalid input", () {
+ expect(() => new Timeout.parse(".d"), throwsFormatException);
+ expect(() => new Timeout.parse("d"), throwsFormatException);
+ expect(() => new Timeout.parse("ad"), throwsFormatException);
+ expect(() => new Timeout.parse("1z"), throwsFormatException);
+ expect(() => new Timeout.parse("1u"), throwsFormatException);
+ expect(() => new Timeout.parse("1d5x"), throwsFormatException);
+ expect(() => new Timeout.parse("1d*5m"), throwsFormatException);
+ });
+ });
+ });
+}
diff --git a/test/runner/runner_test.dart b/test/runner/runner_test.dart
index 6bacd10..8a69547 100644
--- a/test/runner/runner_test.dart
+++ b/test/runner/runner_test.dart
@@ -344,27 +344,6 @@
test.shouldExit(1);
});
- test("respects top-level @Timeout declarations", () {
- d.file("test.dart", '''
-@Timeout(const Duration(seconds: 0))
-
-import 'dart:async';
-
-import 'package:test/test.dart';
-
-void main() {
- test("timeout", () {});
-}
-''').create();
-
- var test = runTest(["test.dart"]);
- test.stdout.expect(containsInOrder([
- "Test timed out after 0 seconds.",
- "-1: Some tests failed."
- ]));
- test.shouldExit(1);
- });
-
test("respects top-level @Skip declarations", () {
d.file("test.dart", '''
@Skip()
diff --git a/test/runner/timeout_test.dart b/test/runner/timeout_test.dart
new file mode 100644
index 0000000..e1c8a5f
--- /dev/null
+++ b/test/runner/timeout_test.dart
@@ -0,0 +1,91 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+
+import 'dart:io';
+import 'dart:math' as math;
+
+import 'package:scheduled_test/descriptor.dart' as d;
+import 'package:scheduled_test/scheduled_stream.dart';
+import 'package:scheduled_test/scheduled_test.dart';
+import 'package:test/src/util/exit_codes.dart' as exit_codes;
+
+import '../io.dart';
+
+void main() {
+ useSandbox();
+
+ test("respects top-level @Timeout declarations", () {
+ d.file("test.dart", '''
+@Timeout(const Duration(seconds: 0))
+
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+ test("timeout", () async {
+ await new Future.delayed(Duration.ZERO);
+ });
+}
+''').create();
+
+ var test = runTest(["test.dart"]);
+ test.stdout.expect(containsInOrder([
+ "Test timed out after 0 seconds.",
+ "-1: Some tests failed."
+ ]));
+ test.shouldExit(1);
+ });
+
+ test("respects the --timeout flag", () {
+ d.file("test.dart", '''
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+ test("timeout", () async {
+ await new Future.delayed(Duration.ZERO);
+ });
+}
+''').create();
+
+ var test = runTest(["--timeout=0s", "test.dart"]);
+ test.stdout.expect(containsInOrder([
+ "Test timed out after 0 seconds.",
+ "-1: Some tests failed."
+ ]));
+ test.shouldExit(1);
+ });
+
+ test("the --timeout flag applies on top of the default 30s timeout", () {
+ d.file("test.dart", '''
+import 'dart:async';
+
+import 'package:test/test.dart';
+
+void main() {
+ test("no timeout", () async {
+ await new Future.delayed(new Duration(milliseconds: 250));
+ });
+
+ test("timeout", () async {
+ await new Future.delayed(new Duration(milliseconds: 750));
+ });
+}
+''').create();
+
+ // This should make the timeout about 500ms, which should cause exactly one
+ // test to fail.
+ var test = runTest(["--timeout=0.016x", "test.dart"]);
+ test.stdout.expect(containsInOrder([
+ "Test timed out after 0.4 seconds.",
+ "-1: Some tests failed."
+ ]));
+ test.shouldExit(1);
+ });
+}
+
