Implement VM coverage gathering (#1088)
Open to feedback on this. Here's how it works at the moment:
There's a new `--coverage` option that you can use to specify the output directory for your coverage. It will create a new output directory if one doesn't already exist.
The tests are run through the engine, and then get spit out to a coverage helper function. That coverage helper function:
**A)** Checks if it's a VM suite
**B)** Gathers coverage if it is
**C)** Outputs coverage to `${suite path}.vm.json`
So, for example, when I run:
```bash
pub run test --coverage hello_world test/vm/simple_repo_test.dart
```
A new file gets added at `hello_world/test/vm/simple_repo_test.dart.vm.json` with the coverage results!diff --git a/.travis.yml b/.travis.yml
index a58b8a6..854d577 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -18,14 +18,9 @@
env: PKGS="pkgs/test pkgs/test_api pkgs/test_core"
script: ./tool/travis.sh dartfmt dartanalyzer_0
- stage: analyze_and_format
- name: "SDK: 2.2.0; PKG: pkgs/test; TASKS: `dartanalyzer --fatal-warnings .`"
+ name: "SDK: 2.2.0; PKG: pkgs/test, pkgs/test_api, pkgs/test_core; TASKS: `dartanalyzer --fatal-warnings .`"
dart: "2.2.0"
- env: PKGS="pkgs/test"
- script: ./tool/travis.sh dartanalyzer_1
- - stage: analyze_and_format
- name: "SDK: 2.1.0; PKGS: pkgs/test_api, pkgs/test_core; TASKS: `dartanalyzer --fatal-warnings .`"
- dart: "2.1.0"
- env: PKGS="pkgs/test_api pkgs/test_core"
+ env: PKGS="pkgs/test pkgs/test_api pkgs/test_core"
script: ./tool/travis.sh dartanalyzer_1
- stage: unit_test
name: "SDK: dev; PKG: pkgs/test; TASKS: `xvfb-run -s \"-screen 0 1024x768x24\" pub run test --preset travis --total-shards 5 --shard-index 0`"
diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md
index f2acb13..9278b43 100644
--- a/pkgs/test/CHANGELOG.md
+++ b/pkgs/test/CHANGELOG.md
@@ -1,3 +1,7 @@
+## 1.9.0
+
+* Implement code coverage collection for VM based tests
+
## 1.8.0
* Expose the previously hidden sharding arguments
diff --git a/pkgs/test/README.md b/pkgs/test/README.md
index 0063505..5bd7e1d 100644
--- a/pkgs/test/README.md
+++ b/pkgs/test/README.md
@@ -2,6 +2,8 @@
* [Writing Tests](#writing-tests)
* [Running Tests](#running-tests)
+ * [Sharding Tests](#sharding-tests)
+ * [Collecting Code Coverage](#collecting-code-coverage)
* [Restricting Tests to Certain Platforms](#restricting-tests-to-certain-platforms)
* [Platform Selectors](#platform-selectors)
* [Running Tests on Node.js](#running-tests-on-nodejs)
@@ -157,6 +159,7 @@
tests on both platforms with a single command: `pub run test -p "chrome,vm"
path/to/test.dart`.
+### Sharding Tests
Tests can also be sharded with the `--total-shards` and `--shard-index` arguments,
allowing you to split up your test suites and run them separately. For example,
if you wanted to run 3 shards of your test suite, you could run them as follows:
@@ -167,6 +170,20 @@
pub run test --total-shards 3 --shard-index 2 path/to/test.dart
```
+### Collecting Code Coverage
+To collect code coverage, you can run tests with the `--coverage <directory>`
+argument. The directory specified can be an absolute or relative path.
+If a directory does not exist at the path specified, a directory will be
+created. If a directory does exist, files may be overwritten with the latest
+coverage data, if they conflict.
+
+This option will enable code coverage collection on a suite-by-suite basis,
+and the resulting coverage files will be outputted in the directory specified.
+The files can then be formatted using the `package:coverage`
+`format_coverage` executable.
+
+Coverage gathering is currently only implemented for tests run in the Dart VM.
+
### Restricting Tests to Certain Platforms
Some test files only make sense to run on particular platforms. They may use
diff --git a/pkgs/test/pubspec.yaml b/pkgs/test/pubspec.yaml
index 4637b4a..c94a1ad 100644
--- a/pkgs/test/pubspec.yaml
+++ b/pkgs/test/pubspec.yaml
@@ -1,5 +1,5 @@
name: test
-version: 1.8.0
+version: 1.9.0
author: Dart Team <misc@dartlang.org>
description: A full featured library for writing and running Dart tests.
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test
@@ -32,7 +32,7 @@
yaml: ^2.0.0
# Use an exact version until the test_api and test_core package are stable.
test_api: 0.2.8
- test_core: 0.2.10
+ test_core: 0.2.11
dev_dependencies:
fake_async: ^1.0.0
diff --git a/pkgs/test/test/runner/coverage_test.dart b/pkgs/test/test/runner/coverage_test.dart
new file mode 100644
index 0000000..7bdaf66
--- /dev/null
+++ b/pkgs/test/test/runner/coverage_test.dart
@@ -0,0 +1,50 @@
+// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn("vm")
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:test_descriptor/test_descriptor.dart' as d;
+
+import 'package:path/path.dart' as p;
+import 'package:test/test.dart';
+
+import '../io.dart';
+
+void main() {
+ group("with the --coverage flag,", () {
+ test("gathers coverage for VM tests", () async {
+ await d.file("test.dart", """
+ import 'package:test/test.dart';
+
+ void main() {
+ test("test 1", () {
+ expect(true, isTrue);
+ });
+ }
+ """).create();
+
+ final coverageDirectory =
+ Directory(p.join(Directory.current.path, "test_coverage"));
+ expect(await coverageDirectory.exists(), isFalse,
+ reason:
+ 'Coverage directory exists, cannot safely run coverage tests. Delete the ${coverageDirectory.path} directory to fix.');
+
+ var test =
+ await runTest(["--coverage", coverageDirectory.path, "test.dart"]);
+ expect(test.stdout, emitsThrough(contains("+1: All tests passed!")));
+ await test.shouldExit(0);
+
+ final coverageFile =
+ File(p.join(coverageDirectory.path, "test.dart.vm.json"));
+ final coverage = await coverageFile.readAsString();
+ final jsonCoverage = json.decode(coverage);
+ expect(jsonCoverage['coverage'], isNotEmpty);
+
+ await coverageDirectory.delete(recursive: true);
+ });
+ });
+}
diff --git a/pkgs/test/test/runner/runner_test.dart b/pkgs/test/test/runner/runner_test.dart
index 483d942..2359b0b 100644
--- a/pkgs/test/test/runner/runner_test.dart
+++ b/pkgs/test/test/runner/runner_test.dart
@@ -95,6 +95,9 @@
Currently only supported for browser tests.
--debug Runs the VM and Chrome tests in debug mode.
+ --coverage=<directory> Gathers coverage and outputs it to the specified directory.
+ Implies --debug.
+
--[no-]chain-stack-traces Chained stack traces to provide greater exception details
especially for asynchronous code. It may be useful to disable
to provide improved test performance but at the cost of
diff --git a/pkgs/test/test/utils.dart b/pkgs/test/test/utils.dart
index eadf3d6..7ebb534 100644
--- a/pkgs/test/test/utils.dart
+++ b/pkgs/test/test/utils.dart
@@ -185,7 +185,7 @@
}
/// Runs [body] with a declarer and returns an engine that runs those tests.
-Engine declareEngine(void body(), {bool runSkipped = false}) {
+Engine declareEngine(void body(), {bool runSkipped = false, String coverage}) {
var declarer = Declarer()..declare(body);
return Engine.withSuites([
RunnerSuite(
@@ -193,7 +193,7 @@
SuiteConfiguration(runSkipped: runSkipped),
declarer.build(),
suitePlatform)
- ]);
+ ], coverage: coverage);
}
/// Returns a [RunnerSuite] with a default environment and configuration.
diff --git a/pkgs/test_api/pubspec.yaml b/pkgs/test_api/pubspec.yaml
index d04789e..e79b863 100644
--- a/pkgs/test_api/pubspec.yaml
+++ b/pkgs/test_api/pubspec.yaml
@@ -5,7 +5,7 @@
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_api
environment:
- sdk: ">=2.1.0 <3.0.0"
+ sdk: ">=2.2.0 <3.0.0"
dependencies:
async: ^2.0.0
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index 4c81fc0..e25a3c4 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -1,6 +1,10 @@
+## 0.2.11
+
+* Implement code coverage gathering for VM tests.
+
## 0.2.10
-Add a `--debug` argument for running the VM/Chrome in debug mode.
+* Add a `--debug` argument for running the VM/Chrome in debug mode.
## 0.2.9+2
diff --git a/pkgs/test_core/lib/src/runner.dart b/pkgs/test_core/lib/src/runner.dart
index 9ea4952..5712035 100644
--- a/pkgs/test_core/lib/src/runner.dart
+++ b/pkgs/test_core/lib/src/runner.dart
@@ -71,7 +71,8 @@
/// Creates a new runner based on [configuration].
factory Runner(Configuration config) => config.asCurrent(() {
- var engine = Engine(concurrency: config.concurrency);
+ var engine =
+ Engine(concurrency: config.concurrency, coverage: config.coverage);
var reporterDetails = allReporters[config.reporter];
return Runner._(engine, reporterDetails.factory(config, engine));
@@ -106,6 +107,10 @@
'`DART_VM_OPTIONS=-DSILENT_OBSERVATORY=true`.');
}
+ if (_config.coverage != null) {
+ await Directory(_config.coverage).create(recursive: true);
+ }
+
bool success;
if (_config.pauseAfterLoad) {
success = await _loadThenPause(suites);
diff --git a/pkgs/test_core/lib/src/runner/configuration.dart b/pkgs/test_core/lib/src/runner/configuration.dart
index cf18a57..f1df78f 100644
--- a/pkgs/test_core/lib/src/runner/configuration.dart
+++ b/pkgs/test_core/lib/src/runner/configuration.dart
@@ -52,9 +52,13 @@
final bool _pauseAfterLoad;
/// Whether to run browsers in their respective debug modes
- bool get debug => pauseAfterLoad || (_debug ?? false);
+ bool get debug => pauseAfterLoad || (_debug ?? false) || _coverage != null;
final bool _debug;
+ /// The output folder for coverage gathering
+ String get coverage => _coverage;
+ final String _coverage;
+
/// The path to the file from which to load more configuration information.
///
/// This is *not* resolved automatically.
@@ -219,6 +223,7 @@
String configurationPath,
String dart2jsPath,
String reporter,
+ String coverage,
int pubServePort,
int concurrency,
int shardIndex,
@@ -264,6 +269,7 @@
configurationPath: configurationPath,
dart2jsPath: dart2jsPath,
reporter: reporter,
+ coverage: coverage,
pubServePort: pubServePort,
concurrency: concurrency,
shardIndex: shardIndex,
@@ -322,6 +328,7 @@
String configurationPath,
String dart2jsPath,
String reporter,
+ String coverage,
int pubServePort,
int concurrency,
this.shardIndex,
@@ -344,6 +351,7 @@
_configurationPath = configurationPath,
_dart2jsPath = dart2jsPath,
_reporter = reporter,
+ _coverage = coverage,
pubServeUrl = pubServePort == null
? null
: Uri.parse("http://localhost:$pubServePort"),
@@ -465,6 +473,7 @@
configurationPath: other._configurationPath ?? _configurationPath,
dart2jsPath: other._dart2jsPath ?? _dart2jsPath,
reporter: other._reporter ?? _reporter,
+ coverage: other._coverage ?? _coverage,
pubServePort: (other.pubServeUrl ?? pubServeUrl)?.port,
concurrency: other._concurrency ?? _concurrency,
shardIndex: other.shardIndex ?? shardIndex,
diff --git a/pkgs/test_core/lib/src/runner/configuration/args.dart b/pkgs/test_core/lib/src/runner/configuration/args.dart
index 6c244b2..e0150f3 100644
--- a/pkgs/test_core/lib/src/runner/configuration/args.dart
+++ b/pkgs/test_core/lib/src/runner/configuration/args.dart
@@ -91,6 +91,10 @@
negatable: false);
parser.addFlag("debug",
help: 'Runs the VM and Chrome tests in debug mode.', negatable: false);
+ parser.addOption("coverage",
+ help: 'Gathers coverage and outputs it to the specified directory.\n'
+ 'Implies --debug.',
+ valueHelp: 'directory');
parser.addFlag("chain-stack-traces",
help: 'Chained stack traces to provide greater exception details\n'
'especially for asynchronous code. It may be useful to disable\n'
@@ -214,6 +218,7 @@
dart2jsArgs: _ifParsed('dart2js-args'),
precompiledPath: _ifParsed('precompiled'),
reporter: _ifParsed('reporter'),
+ coverage: _ifParsed('coverage'),
pubServePort: _parseOption('pub-serve', int.parse),
concurrency: _parseOption('concurrency', int.parse),
shardIndex: shardIndex,
diff --git a/pkgs/test_core/lib/src/runner/engine.dart b/pkgs/test_core/lib/src/runner/engine.dart
index 647c9a5..11ae315 100644
--- a/pkgs/test_core/lib/src/runner/engine.dart
+++ b/pkgs/test_core/lib/src/runner/engine.dart
@@ -4,9 +4,13 @@
import 'dart:async';
import 'dart:collection';
+import 'dart:convert';
+import 'dart:io';
import 'package:async/async.dart' hide Result;
import 'package:collection/collection.dart';
+import 'package:coverage/coverage.dart';
+import 'package:path/path.dart' as p;
import 'package:pedantic/pedantic.dart';
import 'package:pool/pool.dart';
@@ -66,6 +70,9 @@
/// `false` if the tests finished running before close was called.
bool _closedBeforeDone;
+ /// The coverage output directory.
+ String _coverage;
+
/// A pool that limits the number of test suites running concurrently.
final Pool _runPool;
@@ -214,9 +221,10 @@
/// [concurrency] controls how many suites are run at once, and defaults to 1.
/// [maxSuites] controls how many suites are *loaded* at once, and defaults to
/// four times [concurrency].
- Engine({int concurrency, int maxSuites})
+ Engine({int concurrency, int maxSuites, String coverage})
: _runPool = Pool(concurrency ?? 1),
- _loadPool = Pool(maxSuites ?? (concurrency ?? 1) * 2) {
+ _loadPool = Pool(maxSuites ?? (concurrency ?? 1) * 2),
+ _coverage = coverage {
_group.future.then((_) {
_onTestStartedGroup.close();
_onSuiteStartedController.close();
@@ -233,8 +241,9 @@
///
/// [concurrency] controls how many suites are run at once. If [runSkipped] is
/// `true`, skipped tests will be run as though they weren't skipped.
- factory Engine.withSuites(List<RunnerSuite> suites, {int concurrency}) {
- var engine = Engine(concurrency: concurrency);
+ factory Engine.withSuites(List<RunnerSuite> suites,
+ {int concurrency, String coverage}) {
+ var engine = Engine(concurrency: concurrency, coverage: coverage);
for (var suite in suites) {
engine.suiteSink.add(suite);
}
@@ -279,6 +288,7 @@
if (_closed) return;
await _runGroup(controller, controller.liveSuite.suite.group, []);
controller.noMoreLiveTests();
+ await _gatherCoverage(controller);
loadResource.allowRelease(() => controller.close());
});
}());
@@ -293,6 +303,29 @@
return success;
}
+ Future<Null> _gatherCoverage(LiveSuiteController controller) async {
+ if (_coverage == null) return;
+
+ final RunnerSuite suite = controller.liveSuite.suite;
+
+ if (!suite.platform.runtime.isDartVM) return;
+
+ final String isolateId =
+ Uri.parse(suite.environment.observatoryUrl.fragment)
+ .queryParameters['isolateId'];
+
+ final cov = await collect(
+ suite.environment.observatoryUrl, false, false, false, Set(),
+ isolateIds: {isolateId});
+
+ final outfile = File(p.join('$_coverage', '${suite.path}.vm.json'))
+ ..createSync(recursive: true);
+ final IOSink out = outfile.openWrite();
+ out.write(json.encode(cov));
+ await out.flush();
+ await out.close();
+ }
+
/// Runs all the entries in [group] in sequence.
///
/// [suiteController] is the controller fo the suite that contains [group].
diff --git a/pkgs/test_core/pubspec.yaml b/pkgs/test_core/pubspec.yaml
index cf2e86b..9aec1fb 100644
--- a/pkgs/test_core/pubspec.yaml
+++ b/pkgs/test_core/pubspec.yaml
@@ -1,11 +1,11 @@
name: test_core
-version: 0.2.10
+version: 0.2.11
author: Dart Team <misc@dartlang.org>
description: A basic library for writing tests and running them on the VM.
homepage: https://github.com/dart-lang/test/blob/master/pkgs/test_core
environment:
- sdk: ">=2.1.0 <3.0.0"
+ sdk: ">=2.2.0 <3.0.0"
dependencies:
analyzer: ">=0.36.0 <0.39.0"
@@ -13,6 +13,7 @@
args: ^1.4.0
boolean_selector: ^1.0.0
collection: ^1.8.0
+ coverage: ^0.13.3
glob: ^1.0.0
io: ^0.3.0
meta: ^1.1.5
