Add capabilities for building direct test runners (#1332)
Towards #1310, #1328
Add `directRunTest` to configure a reporter and run tests directly in
the same isolate.
Add `enumerateTestCases` to collect test names without running them, and
`directRunSingleTest` to run a specific test by its full name. These
APIs ensure the uniqueness of test names to avoid ambiguity. This
restriction may be spread to tests run through the normal test runner as
well.
- Add `fullTestName` option on `Declarer`. When used, only the test (or
tests if uniqueness is not checked separately) will be considered as a
test case.
- Add `directRunTest`, `enumerateTestCases`, and `directRunSingleTest`
APIs. These are kept under a `lib/src/` import for now, and any other
package that uses these APIs should pin to a specific version of
`package:test_core`. The details of these APIs might change without a
major version bump.
diff --git a/pkgs/test_api/CHANGELOG.md b/pkgs/test_api/CHANGELOG.md
index f8ea8c5..44627be 100644
--- a/pkgs/test_api/CHANGELOG.md
+++ b/pkgs/test_api/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 0.2.19-nullsafety.3-dev
+
+* Add capability to filter to a single exact test name in `Declarer`.
+
## 0.2.19-nullsafety.2
* Allow `2.10` stable and `2.11.0-dev` SDKs.
diff --git a/pkgs/test_api/lib/src/backend/declarer.dart b/pkgs/test_api/lib/src/backend/declarer.dart
index 9ff1de7..9211776 100644
--- a/pkgs/test_api/lib/src/backend/declarer.dart
+++ b/pkgs/test_api/lib/src/backend/declarer.dart
@@ -91,6 +91,17 @@
/// Whether any tests and/or groups have been flagged as solo.
bool get _solo => _soloEntries.isNotEmpty;
+ /// An exact full test name to match.
+ ///
+ /// When non-null only tests with exactly this name will be considered. The
+ /// full test name is the combination of the test case name with all group
+ /// prefixes. All other tests, including their metadata like `solo`, is
+ /// ignored. Uniqueness is not guaranteed so this may match more than one
+ /// test.
+ ///
+ /// Groups which are not a strict prefix of this name will be ignored.
+ final String? _fullTestName;
+
/// The current zone-scoped declarer.
static Declarer? get current => Zone.current[#test.declarer] as Declarer?;
@@ -113,7 +124,8 @@
{Metadata? metadata,
Set<String>? platformVariables,
bool collectTraces = false,
- bool noRetry = false})
+ bool noRetry = false,
+ String? fullTestName})
: this._(
null,
null,
@@ -121,10 +133,19 @@
platformVariables ?? const UnmodifiableSetView.empty(),
collectTraces,
null,
- noRetry);
+ noRetry,
+ fullTestName);
- Declarer._(this._parent, this._name, this._metadata, this._platformVariables,
- this._collectTraces, this._trace, this._noRetry);
+ Declarer._(
+ this._parent,
+ this._name,
+ this._metadata,
+ this._platformVariables,
+ this._collectTraces,
+ this._trace,
+ this._noRetry,
+ this._fullTestName,
+ );
/// Runs [body] with this declarer as [Declarer.current].
///
@@ -143,6 +164,11 @@
bool solo = false}) {
_checkNotBuilt('test');
+ final fullName = _prefix(name);
+ if (_fullTestName != null && fullName != _fullTestName) {
+ return;
+ }
+
var newMetadata = Metadata.parse(
testOn: testOn,
timeout: timeout,
@@ -152,8 +178,7 @@
retry: _noRetry ? 0 : retry);
newMetadata.validatePlatformSelectors(_platformVariables);
var metadata = _metadata.merge(newMetadata);
-
- _entries.add(LocalTest(_prefix(name), metadata, () async {
+ _entries.add(LocalTest(fullName, metadata, () async {
var parents = <Declarer>[];
for (Declarer? declarer = this;
declarer != null;
@@ -195,6 +220,11 @@
bool solo = false}) {
_checkNotBuilt('group');
+ final fullTestPrefix = _prefix(name);
+ if (_fullTestName != null && !_fullTestName!.startsWith(fullTestPrefix)) {
+ return;
+ }
+
var newMetadata = Metadata.parse(
testOn: testOn,
timeout: timeout,
@@ -206,8 +236,8 @@
var metadata = _metadata.merge(newMetadata);
var trace = _collectTraces ? Trace.current(2) : null;
- var declarer = Declarer._(this, _prefix(name), metadata, _platformVariables,
- _collectTraces, trace, _noRetry);
+ var declarer = Declarer._(this, fullTestPrefix, metadata,
+ _platformVariables, _collectTraces, trace, _noRetry, _fullTestName);
declarer.declare(() {
// Cast to dynamic to avoid the analyzer complaining about us using the
// result of a void method.
diff --git a/pkgs/test_api/pubspec.yaml b/pkgs/test_api/pubspec.yaml
index 305706a..87c45e9 100644
--- a/pkgs/test_api/pubspec.yaml
+++ b/pkgs/test_api/pubspec.yaml
@@ -1,5 +1,5 @@
name: test_api
-version: 0.2.19-nullsafety.2
+version: 0.2.19-nullsafety.3-dev
description: A library for writing Dart tests.
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_api
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index 8a6a39a..1daf214 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 0.3.12-nullsafety.6-dev
+
+* Add experimental `directRunTests`, `directRunSingle`, and `enumerateTestCases`
+ APIs to enable test runners written around a single executable that can report
+ and run any single test case.
+
## 0.3.12-nullsafety.5
* Allow `2.10` stable and `2.11.0-dev` SDKs.
diff --git a/pkgs/test_core/lib/src/direct_run.dart b/pkgs/test_core/lib/src/direct_run.dart
new file mode 100644
index 0000000..3ed0b12
--- /dev/null
+++ b/pkgs/test_core/lib/src/direct_run.dart
@@ -0,0 +1,138 @@
+// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:collection';
+
+import 'package:path/path.dart' as p;
+import 'package:test_api/backend.dart'; //ignore: deprecated_member_use
+import 'package:test_api/src/backend/declarer.dart'; //ignore: implementation_imports
+import 'package:test_api/src/backend/group.dart'; //ignore: implementation_imports
+import 'package:test_api/src/backend/group_entry.dart'; //ignore: implementation_imports
+import 'package:test_api/src/backend/invoker.dart'; // ignore: implementation_imports
+import 'package:test_api/src/backend/test.dart'; //ignore: implementation_imports
+import 'package:test_api/src/utils.dart'; // ignore: implementation_imports
+
+import 'runner/configuration.dart';
+import 'runner/engine.dart';
+import 'runner/plugin/environment.dart';
+import 'runner/reporter.dart';
+import 'runner/reporter/expanded.dart';
+import 'runner/runner_suite.dart';
+import 'runner/suite.dart';
+import 'util/print_sink.dart';
+
+/// Runs all unskipped test cases declared in [testMain].
+///
+/// Test suite level metadata defined in annotations is not read. No filtering
+/// is applied except for the filtering defined by `solo` or `skip` arguments to
+/// `group` and `test`. Returns [true] if all tests passed.
+Future<bool> directRunTests(FutureOr<void> Function() testMain,
+ {Reporter Function(Engine)? reporterFactory}) =>
+ _directRunTests(testMain, reporterFactory: reporterFactory);
+
+/// Runs a single test declared in [testMain] matched by it's full test name.
+///
+/// There must be exactly one test defined with the name [fullTestName]. Note
+/// that not all tests and groups are checked, so a test case that is not be
+/// intended to be run (due to a `solo` on a different test) may still be run
+/// with this API. Only the test names returned by [enumerateTestCases] should
+/// be used to prevent running skipped tests.
+///
+/// Return [true] if the test passes.
+///
+/// If there are no tests matching [fullTestName] a [MissingTestException] is
+/// thrown. If there is more than one test with the name [fullTestName] they
+/// will both be run, then a [DuplicateTestnameException] will be thrown.
+Future<bool> directRunSingleTest(
+ FutureOr<void> Function() testMain, String fullTestName,
+ {Reporter Function(Engine)? reporterFactory}) =>
+ _directRunTests(testMain,
+ reporterFactory: reporterFactory, fullTestName: fullTestName);
+
+Future<bool> _directRunTests(FutureOr<void> Function() testMain,
+ {Reporter Function(Engine)? reporterFactory, String? fullTestName}) async {
+ reporterFactory ??= (engine) => ExpandedReporter.watch(engine, PrintSink(),
+ color: Configuration.empty.color, printPath: false, printPlatform: false);
+ final declarer = Declarer(fullTestName: fullTestName);
+ await declarer.declare(testMain);
+
+ final suite = RunnerSuite(const PluginEnvironment(), SuiteConfiguration.empty,
+ declarer.build(), SuitePlatform(Runtime.vm, os: currentOSGuess),
+ path: p.prettyUri(Uri.base));
+
+ final engine = Engine()
+ ..suiteSink.add(suite)
+ ..suiteSink.close();
+
+ reporterFactory(engine);
+
+ final success = await runZoned(() => Invoker.guard(engine.run),
+ zoneValues: {#test.declarer: declarer});
+
+ if (fullTestName != null) {
+ final testCount = engine.liveTests.length;
+ if (testCount > 1) {
+ throw DuplicateTestNameException(fullTestName);
+ }
+ if (testCount == 0) {
+ throw MissingTestException(fullTestName);
+ }
+ }
+ return success!;
+}
+
+/// Runs [testMain] and returns the names of all declared tests.
+///
+/// Test names declared must be unique. If any test repeats the full name,
+/// including group prefixes, of a prior test a [DuplicateTestNameException]
+/// will be thrown.
+///
+/// Skipped tests are ignored.
+Future<Set<String>> enumerateTestCases(
+ FutureOr<void> Function() testMain) async {
+ final declarer = Declarer();
+ await declarer.declare(testMain);
+
+ final toVisit = Queue<GroupEntry>.of([declarer.build()]);
+ final allTestNames = <String>{};
+ final unskippedTestNames = <String>{};
+ while (toVisit.isNotEmpty) {
+ final current = toVisit.removeLast();
+ if (current is Group) {
+ toVisit.addAll(current.entries.reversed);
+ } else if (current is Test) {
+ if (!allTestNames.add(current.name)) {
+ throw DuplicateTestNameException(current.name);
+ }
+ if (current.metadata.skip) continue;
+ unskippedTestNames.add(current.name);
+ } else {
+ throw StateError('Unandled Group Entry: ${current.runtimeType}');
+ }
+ }
+ return unskippedTestNames;
+}
+
+/// An exception thrown when two test cases in the same test suite (same `main`)
+/// have an identical name.
+class DuplicateTestNameException implements Exception {
+ final String name;
+ DuplicateTestNameException(this.name);
+
+ @override
+ String toString() => 'A test with the name "$name" was already declared. '
+ 'Test cases must have unique names.';
+}
+
+/// An exception thrown when a specific test was requested by name that does not
+/// exist.
+class MissingTestException implements Exception {
+ final String name;
+ MissingTestException(this.name);
+
+ @override
+ String toString() =>
+ 'A test with the name "$name" was not declared in the test suite.';
+}
diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml
index e535d45..d0ad6ca 100644
--- a/pkgs/test_core/pubspec.yaml
+++ b/pkgs/test_core/pubspec.yaml
@@ -1,5 +1,5 @@
name: test_core
-version: 0.3.12-nullsafety.5
+version: 0.3.12-nullsafety.6-dev
description: A basic library for writing tests and running them on the VM.
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_core
@@ -31,7 +31,7 @@
# matcher is tightly constrained by test_api
matcher: any
# Use an exact version until the test_api package is stable.
- test_api: 0.2.19-nullsafety.2
+ test_api: 0.2.19-nullsafety.3
dependency_overrides:
test_api:
