Allow running tests by line and column (#1604)
Fixes https://github.com/dart-lang/test/issues/1579
- Allows querying tests by line & col using test path query strings
- Matches if any frame in the test stack trace matches up (this allows for lots of flexibility in terms of testing styles and parameterized tests).
- If a stack trace formatter is set up for the remote platform, it will use it to format the `trace` sent back for groups/tests that are discovered (enabling browser/node support).
diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md
index 7d093a5..ecdaee2 100644
--- a/pkgs/test/CHANGELOG.md
+++ b/pkgs/test/CHANGELOG.md
@@ -1,9 +1,11 @@
## 1.19.0-dev
-* Support query parameters `name` and `full-name` on test paths, which will
- apply the filters to only those test suites.
+* Support query parameters `name`, `full-name`, `line`, and `col` on test paths,
+ which will apply the filters to only those test suites.
* All specified filters must match for a test to run.
* Global filters (ie: `--name`) are also still respected and must match.
+ * The `line` and `col` will match if any frame from the test trace matches
+ (the test trace is the current stack trace where `test` is invoked).
* Give a better exception when using `markTestSkipped` outside of a test.
## 1.18.2
diff --git a/pkgs/test/README.md b/pkgs/test/README.md
index 789cc5f..c63d7fc 100644
--- a/pkgs/test/README.md
+++ b/pkgs/test/README.md
@@ -173,10 +173,6 @@
will be run. You can also use the `-N` flag to run tests whose names contain a
plain-text string.
-Alternatively, you can filter tests by name only for a specific file/directory
-by specifying a query on a path: `dart test "path/to/test.dart?name=test name"`.
-That ensures that the name filters applies to one path but not others.
-
By default, tests are run in the Dart VM, but you can run them in the browser as
well by passing `dart test -p chrome path/to/test.dart`. `test` will take
care of starting the browser and loading the tests, and all the results will be
@@ -184,6 +180,34 @@
tests on both platforms with a single command: `dart test -p "chrome,vm"
path/to/test.dart`.
+### Test Path Queries
+
+Some query parameters are supported on test paths, which allow you to filter the
+tests that will run within just those paths. These filters are merged with any
+global options that are passed, and all filters must match for a test to be ran.
+
+- **name**: Works the same as `--name` (simple contains check).
+ - This is the only option that supports more than one entry.
+- **full-name**: Requires an exact match for the name of the test.
+- **line**: Matches any test that originates from this line in the test suite.
+- **col**: Matches any test that originates from this column in the test suite.
+
+**Example Usage**: `dart test "path/to/test.dart?line=10&col=2"`
+
+#### Line/Col Matching Semantics
+
+The `line` and `col` filters match against the current stack trace taken from
+the invocation to the `test` function, and are considered a match if
+**any frame** in the trace meets **all** of the following criteria:
+
+* The URI of the frame matches the root test suite uri.
+ * This means it will not match lines from imported libraries.
+* If both `line` and `col` are passed, both must match **the same frame**.
+* The specific `line` and `col` to be matched are defined by the tools creating
+ the stack trace. This generally means they are 1 based and not 0 based, but
+ this package is not in control of the exact semantics and they may vary based
+ on platform implementations.
+
### Sharding Tests
Tests can also be sharded with the `--total-shards` and `--shard-index` arguments,
allowing you to split up your test suites and run them separately. For example,
diff --git a/pkgs/test/test/runner/line_and_col_test.dart b/pkgs/test/test/runner/line_and_col_test.dart
new file mode 100644
index 0000000..94b5a83
--- /dev/null
+++ b/pkgs/test/test/runner/line_and_col_test.dart
@@ -0,0 +1,365 @@
+// Copyright (c) 2021, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn('vm')
+
+import 'package:test_descriptor/test_descriptor.dart' as d;
+
+import 'package:test_core/src/util/exit_codes.dart' as exit_codes;
+import 'package:test/test.dart';
+
+import '../io.dart';
+
+void main() {
+ setUpAll(precompileTestExecutable);
+
+ group('with test.dart?line=<line> query', () {
+ test('selects test with the matching line', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ test("b", () => throw TestFailure("oh no"));
+ test("c", () {});
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=6']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('selects multiple tests on the same line', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {}); test("b", () {});
+ test("c", () => throw TestFailure("oh no"));
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=4']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+2: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('selects groups with a matching line', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ group("a", () {
+ test("b", () {});
+ });
+ group("b", () {
+ test("b", () => throw TestFailure("oh no"));
+ });
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=4']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('No matching tests', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=1']);
+
+ expect(
+ test.stderr,
+ emitsThrough(contains('No tests were found.')),
+ );
+
+ await test.shouldExit(exit_codes.noTestsRan);
+ });
+
+ test('allows the line anywhere in the stack trace', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void runTest(String name) {
+ test(name, () {});
+ }
+
+ void main() {
+ runTest("a");
+ test("b", () {});
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=8']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+ });
+
+ group('with test.dart?col=<col> query', () {
+ test('selects single test with the matching column', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ test("b", () => throw TestFailure("oh no"));
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?col=11']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('selects multiple tests starting on the same column', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ test("b", () {});
+ test("c", () => throw TestFailure("oh no"));
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?col=11']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+2: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('selects groups with a matching column', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ group("a", () {
+ test("b", () {});
+ });
+ group("b", () {
+ test("b", () => throw TestFailure("oh no"));
+ });
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?col=11']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('No matching tests', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?col=1']);
+
+ expect(
+ test.stderr,
+ emitsThrough(contains('No tests were found.')),
+ );
+
+ await test.shouldExit(exit_codes.noTestsRan);
+ });
+
+ test('allows the col anywhere in the stack trace', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void runTest(String name) {
+ test(name, () {});
+ }
+
+ void main() {
+ runTest("a");
+ test("b", () => throw TestFailure("oh no"));
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?col=13']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+ });
+
+ group('with test.dart?line=<line>&col=<col> query', () {
+ test('selects test with the matching line and col in the same frame',
+ () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ void runTests() {
+ test("a", () {});test("b", () => throw TestFailure("oh no"));
+ }
+ runTests();
+ test("c", () => throw TestFailure("oh no"));
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=5&col=11']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('selects group with the matching line and col', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ group("a", () {
+ test("b", () {});
+ test("c", () {});
+ });
+ group("d", () {
+ test("e", () => throw TestFailure("oh no"));
+ });
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=4&col=11']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+2: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('no matching tests - col doesnt match', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=4&col=1']);
+
+ expect(
+ test.stderr,
+ emitsThrough(contains('No tests were found.')),
+ );
+
+ await test.shouldExit(exit_codes.noTestsRan);
+ });
+
+ test('no matching tests - line doesnt match', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=1&col=11']);
+
+ expect(
+ test.stderr,
+ emitsThrough(contains('No tests were found.')),
+ );
+
+ await test.shouldExit(exit_codes.noTestsRan);
+ });
+
+ test('supports browser tests', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ test("b", () => throw TestFailure("oh no"));
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=4&col=11', '-p', 'chrome']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+
+ test('supports node tests', () async {
+ await d.file('test.dart', '''
+ import 'package:test/test.dart';
+
+ void main() {
+ test("a", () {});
+ test("b", () => throw TestFailure("oh no"));
+ }
+ ''').create();
+
+ var test = await runTest(['test.dart?line=4&col=11', '-p', 'node']);
+
+ expect(
+ test.stdout,
+ emitsThrough(contains('+1: All tests passed!')),
+ );
+
+ await test.shouldExit(0);
+ });
+ });
+}
diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart
index 52661b3..e14fa51 100644
--- a/pkgs/test/test/utils.dart
+++ b/pkgs/test/test/utils.dart
@@ -227,6 +227,8 @@
BooleanSelector? excludeTags,
Map<BooleanSelector, SuiteConfiguration>? tags,
Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+ int? line,
+ int? col,
// Test-level configuration
Timeout? timeout,
@@ -250,6 +252,8 @@
excludeTags: excludeTags,
tags: tags,
onPlatform: onPlatform,
+ line: line,
+ col: col,
timeout: timeout,
verboseTrace: verboseTrace,
chainStackTraces: chainStackTraces,
diff --git a/pkgs/test_api/CHANGELOG.md b/pkgs/test_api/CHANGELOG.md
index ef875ca..0c5cd35 100644
--- a/pkgs/test_api/CHANGELOG.md
+++ b/pkgs/test_api/CHANGELOG.md
@@ -1,6 +1,8 @@
## 0.4.6-dev
* Give a better exception when using `markTestSkipped` outside of a test.
+* Format stack traces if a formatter is available when serializing tests
+ and groups from the remote listener.
## 0.4.5
diff --git a/pkgs/test_api/lib/src/backend/remote_listener.dart b/pkgs/test_api/lib/src/backend/remote_listener.dart
index 20b00c8..65a4ca5 100644
--- a/pkgs/test_api/lib/src/backend/remote_listener.dart
+++ b/pkgs/test_api/lib/src/backend/remote_listener.dart
@@ -187,7 +187,12 @@
'type': 'group',
'name': group.name,
'metadata': group.metadata.serialize(),
- 'trace': group.trace?.toString(),
+ 'trace': group.trace == null
+ ? null
+ : StackTraceFormatter.current
+ ?.formatStackTrace(group.trace!)
+ .toString() ??
+ group.trace?.toString(),
'setUpAll': _serializeTest(channel, group.setUpAll, parents),
'tearDownAll': _serializeTest(channel, group.tearDownAll, parents),
'entries': group.entries.map((entry) {
@@ -217,7 +222,12 @@
'type': 'test',
'name': test.name,
'metadata': test.metadata.serialize(),
- 'trace': test.trace?.toString(),
+ 'trace': test.trace == null
+ ? null
+ : StackTraceFormatter.current
+ ?.formatStackTrace(test.trace!)
+ .toString() ??
+ test.trace?.toString(),
'channel': testChannel.id
};
}
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index 8585d3c..d735b63 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -1,9 +1,11 @@
## 0.4.6-dev
-* Support query parameters `name` and `full-name` on test paths, which will
- apply the filters to only those test suites.
+* Support query parameters `name`, `full-name`, `line`, and `col` on test paths,
+ which will apply the filters to only those test suites.
* All specified filters must match for a test to run.
* Global filters (ie: `--name`) are also still respected and must match.
+ * The `line` and `col` will match if any frame from the test trace matches
+ (the test trace is the current stack trace where `test` is invoked).
* Support the latest `test_api`.
## 0.4.5
diff --git a/pkgs/test_core/lib/src/runner.dart b/pkgs/test_core/lib/src/runner.dart
index a9d0651..8828019 100644
--- a/pkgs/test_core/lib/src/runner.dart
+++ b/pkgs/test_core/lib/src/runner.dart
@@ -7,6 +7,8 @@
import 'package:async/async.dart';
import 'package:boolean_selector/boolean_selector.dart';
+import 'package:path/path.dart' as p;
+import 'package:stack_trace/stack_trace.dart';
// ignore: deprecated_member_use
import 'package:test_api/backend.dart'
show PlatformSelector, Runtime, SuitePlatform;
@@ -256,14 +258,14 @@
/// suites once they're loaded.
Stream<LoadSuite> _loadSuites() {
return StreamGroup.merge(_config.paths.map((pathConfig) {
- final suiteConfig = pathConfig.testPatterns == null
- ? _config.suiteDefaults
- : _config.suiteDefaults.change(
- patterns: [
- ..._config.suiteDefaults.patterns,
- ...pathConfig.testPatterns!
- ],
- );
+ final suiteConfig = _config.suiteDefaults.change(
+ patterns: [
+ ..._config.suiteDefaults.patterns,
+ ...?pathConfig.testPatterns
+ ],
+ line: pathConfig.line,
+ col: pathConfig.col,
+ );
if (Directory(pathConfig.testPath).existsSync()) {
return _loader.loadDir(pathConfig.testPath, suiteConfig);
@@ -298,6 +300,43 @@
return false;
}
+ // Skip tests that don't start on `line` or `col` if specified.
+ var line = suite.config.line;
+ var col = suite.config.col;
+ if (line != null || col != null) {
+ var trace = test.trace;
+ if (trace == null) {
+ throw StateError(
+ 'Cannot filter by line/column for this test suite, no stack'
+ 'trace available.');
+ }
+ var path = suite.path;
+ if (path == null) {
+ throw StateError(
+ 'Cannot filter by line/column for this test suite, no suite'
+ 'path available.');
+ }
+ var absoluteSuitePath = p.absolute(path);
+
+ bool matchLineAndCol(Frame frame) {
+ if (frame.uri.scheme != 'file' ||
+ frame.uri.toFilePath() != absoluteSuitePath) {
+ return false;
+ }
+ if (line != null && frame.line != line) {
+ return false;
+ }
+ if (col != null && frame.column != col) {
+ return false;
+ }
+ return true;
+ }
+
+ if (!trace.frames.any(matchLineAndCol)) {
+ return false;
+ }
+ }
+
return true;
}));
});
diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart
index 3c061ea..6286db6 100644
--- a/pkgs/test_core/lib/src/runner/configuration.dart
+++ b/pkgs/test_core/lib/src/runner/configuration.dart
@@ -35,6 +35,8 @@
const PathConfiguration({
required this.testPath,
this.testPatterns,
+ this.line,
+ this.col,
});
/// The explicit path to a test suite.
@@ -42,6 +44,12 @@
/// Name filters specific to [testPath].
final List<Pattern>? testPatterns;
+
+ /// Only run tests that originate from this line in the test suite.
+ final int? line;
+
+ /// Only run tests that originate from this column in the test suite.
+ final int? col;
}
/// A class that encapsulates the command-line configuration of the test runner.
@@ -342,6 +350,8 @@
excludeTags: excludeTags,
tags: tags,
onPlatform: onPlatform,
+ line: null, // Only configurable from the command line
+ col: null, // Only configurable from the command line
// Test-level configuration
timeout: timeout,
diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart
index c9cfe9d..71f4da3 100644
--- a/pkgs/test_core/lib/src/runner/configuration/args.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/args.dart
@@ -179,6 +179,8 @@
final names = uri.queryParametersAll['name'];
final fullName = uri.queryParameters['full-name'];
+ final line = uri.queryParameters['line'];
+ final col = uri.queryParameters['col'];
if (names != null && names.isNotEmpty && fullName != null) {
throw FormatException(
@@ -191,6 +193,8 @@
testPatterns: fullName != null
? [RegExp('^${RegExp.escape(fullName)}\$')]
: names?.map((name) => RegExp(name)).toList(),
+ line: line == null ? null : int.parse(line),
+ col: col == null ? null : int.parse(col),
);
}
diff --git a/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart b/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
index 5d4f5ab..2c9ea13 100644
--- a/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
+++ b/pkgs/test_core/lib/src/runner/plugin/platform_helpers.dart
@@ -55,7 +55,9 @@
'asciiGlyphs': Platform.isWindows,
'path': path,
'collectTraces': Configuration.current.reporter == 'json' ||
- Configuration.current.fileReporters.containsKey('json'),
+ Configuration.current.fileReporters.containsKey('json') ||
+ suiteConfig.line != null ||
+ suiteConfig.col != null,
'noRetry': Configuration.current.noRetry,
'foldTraceExcept': Configuration.current.foldTraceExcept.toList(),
'foldTraceOnly': Configuration.current.foldTraceOnly.toList(),
diff --git a/pkgs/test_core/lib/src/runner/suite.dart b/pkgs/test_core/lib/src/runner/suite.dart
index 301d415..115fd7d 100644
--- a/pkgs/test_core/lib/src/runner/suite.dart
+++ b/pkgs/test_core/lib/src/runner/suite.dart
@@ -36,7 +36,9 @@
excludeTags: null,
tags: null,
onPlatform: null,
- metadata: null);
+ metadata: null,
+ line: null,
+ col: null);
/// Whether or not duplicate test (or group) names are allowed within the same
/// test suite.
@@ -132,6 +134,12 @@
});
Set<String>? _knownTags;
+ /// Only run tests that originate from this line in a test file.
+ final int? line;
+
+ /// Only run tests that original from this column in a test file.
+ final int? col;
+
factory SuiteConfiguration(
{required bool? allowDuplicateTestNames,
required bool? allowTestRandomization,
@@ -145,6 +153,8 @@
required BooleanSelector? excludeTags,
required Map<BooleanSelector, SuiteConfiguration>? tags,
required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+ required int? line,
+ required int? col,
// Test-level configuration
required Timeout? timeout,
@@ -168,6 +178,8 @@
excludeTags: excludeTags,
tags: tags,
onPlatform: onPlatform,
+ line: line,
+ col: col,
metadata: Metadata(
timeout: timeout,
verboseTrace: verboseTrace,
@@ -197,6 +209,8 @@
BooleanSelector? excludeTags,
Map<BooleanSelector, SuiteConfiguration>? tags,
Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+ int? line,
+ int? col,
// Test-level configuration
Timeout? timeout,
@@ -220,6 +234,8 @@
excludeTags: excludeTags,
tags: tags,
onPlatform: onPlatform,
+ line: line,
+ col: col,
timeout: timeout,
verboseTrace: verboseTrace,
chainStackTraces: chainStackTraces,
@@ -258,7 +274,9 @@
required BooleanSelector? excludeTags,
required Map<BooleanSelector, SuiteConfiguration>? tags,
required Map<PlatformSelector, SuiteConfiguration>? onPlatform,
- required Metadata? metadata})
+ required Metadata? metadata,
+ required this.line,
+ required this.col})
: _allowDuplicateTestNames = allowDuplicateTestNames,
_allowTestRandomization = allowTestRandomization,
_jsTrace = jsTrace,
@@ -291,6 +309,8 @@
runtimes: null,
includeTags: null,
excludeTags: null,
+ line: null,
+ col: null,
);
/// Returns an unmodifiable copy of [input].
@@ -333,6 +353,8 @@
excludeTags: excludeTags.union(other.excludeTags),
tags: _mergeConfigMaps(tags, other.tags),
onPlatform: _mergeConfigMaps(onPlatform, other.onPlatform),
+ line: other.line ?? line,
+ col: other.col ?? col,
metadata: metadata.merge(other.metadata));
return config._resolveTags();
}
@@ -354,6 +376,8 @@
BooleanSelector? excludeTags,
Map<BooleanSelector, SuiteConfiguration>? tags,
Map<PlatformSelector, SuiteConfiguration>? onPlatform,
+ int? line,
+ int? col,
// Test-level configuration
Timeout? timeout,
@@ -379,6 +403,8 @@
excludeTags: excludeTags ?? this.excludeTags,
tags: tags ?? this.tags,
onPlatform: onPlatform ?? this.onPlatform,
+ line: line ?? this.line,
+ col: col ?? this.col,
metadata: _metadata.change(
timeout: timeout,
verboseTrace: verboseTrace,
