[CFE] Run coverage update on weekly bot and print a diff
This CL will make the weekly bot run tests with coverage and update the
coverage, then printing the diff.
It runs:
* All front_end suits
* python3 tools/test.py -cfasta -mrelease -rnone language
* All "_test.dart" files inside
pkg/{_fe_analyzer_shared,front_end,frontend_server,kernel}/test
We can add more later if we find it useful, although I get something
like
```
$ git diff --stat
[...]
104 files changed, 359 insertions(+), 979 deletions(-)
$ git diff | wc -l
7118
```
so I'm unsure if it will be interesting or just too much data.
Change-Id: Ia0438e618371eff6739b8787accbc8d1425ad548
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/448582
Reviewed-by: Johnni Winther <johnniwinther@google.com>
Commit-Queue: Jens Johansen <jensj@google.com>
diff --git a/pkg/front_end/test/run_all_coverage.dart b/pkg/front_end/test/run_all_coverage.dart
index 9f92387..a80d853 100644
--- a/pkg/front_end/test/run_all_coverage.dart
+++ b/pkg/front_end/test/run_all_coverage.dart
@@ -12,6 +12,22 @@
final Uri repoDirUri = computeRepoDirUri();
Future<void> main() async {
+ Directory coverageTmpDir = await runAllCoverageTests(silent: false);
+
+ // Don't include the not-compiled stuff as we've (mostly) asked the VM to
+ // force compile everything and that the remaining stuff is (mostly) mixins
+ // and const classes etc that shouldn't (necessarily) be compiled but is
+ // potentially covered in other ways.
+ await coverageMerger.mergeFromDirUri(
+ repoDirUri.resolve(".dart_tool/package_config.json"),
+ coverageTmpDir.uri,
+ silent: false,
+ extraCoverageIgnores: const [],
+ extraCoverageBlockIgnores: const [],
+ );
+}
+
+Future<Directory> runAllCoverageTests({required bool silent}) async {
String dartExtension = "";
if (Platform.isWindows) {
dartExtension = ".exe";
@@ -24,19 +40,24 @@
"cfe_coverage",
);
print("Using $coverageTmpDir for coverage.");
- List<List<String>> runThese = [];
- void addSuiteSkipVm(String suitePath) {
- runThese.add([
- dart.toFilePath(),
- "--enable-asserts",
- suitePath,
- "-DskipVm=true",
- "--coverage=${coverageTmpDir.path}/",
- ]);
+ List<List<String>> runTheseSeveralAtATime = [];
+ List<List<String>> runTheseOneAtATime = [];
+ void addSuiteSkipVm(String suitePath, {required int shards}) {
+ for (int shard = 1; shard <= shards; shard++) {
+ runTheseSeveralAtATime.add([
+ dart.toFilePath(),
+ "--enable-asserts",
+ suitePath,
+ "-DskipVm=true",
+ "--coverage=${coverageTmpDir.path}/",
+ "--shards=$shards",
+ "--shard=$shard",
+ ]);
+ }
}
void addWithCoverageArgument(String script) {
- runThese.add([
+ runTheseSeveralAtATime.add([
dart.toFilePath(),
"--enable-asserts",
script,
@@ -44,8 +65,8 @@
]);
}
- addSuiteSkipVm("pkg/front_end/test/strong_suite.dart");
- addSuiteSkipVm("pkg/front_end/test/modular_suite.dart");
+ addSuiteSkipVm("pkg/front_end/test/strong_suite.dart", shards: 4);
+ addSuiteSkipVm("pkg/front_end/test/modular_suite.dart", shards: 1);
addWithCoverageArgument("pkg/front_end/test/messages_suite.dart");
addWithCoverageArgument("pkg/front_end/test/outline_suite.dart");
@@ -68,7 +89,8 @@
addWithCoverageArgument("pkg/front_end/test/spelling_test_src_suite.dart");
addWithCoverageArgument("pkg/front_end/test/compile_platform_coverage.dart");
- runThese.add([
+ // These two each use all available CPUs.
+ runTheseOneAtATime.add([
"python3",
"tools/test.py",
"-cfasta",
@@ -76,7 +98,7 @@
"-rnone",
"language",
]);
- runThese.add([
+ runTheseOneAtATime.add([
dart.toFilePath(),
repoDirUri
.resolve("pkg/front_end/test/run_our_tests_with_coverage.dart")
@@ -88,35 +110,63 @@
);
environment["CFE_COVERAGE"] = "${coverageTmpDir.path}/";
- for (List<String> runThis in runThese) {
+ final int processes = Platform.numberOfProcessors;
+ int processesLeft = processes;
+ int processesRunning = 0;
+ Completer<void> completer = new Completer();
+
+ for (List<String> runThis in runTheseSeveralAtATime) {
+ while (processesLeft <= 0) {
+ await completer.future;
+ }
+ processesLeft--;
+ processesRunning++;
print("Starting $runThis");
- Process p = await Process.start(
- runThis.first,
- runThis.skip(1).toList(),
- environment: environment,
+ unawaited(
+ _run(silent, runThis, environment).then((runExitCode) {
+ print("$runThis finished with exit code $runExitCode.");
+ processesRunning--;
+ processesLeft++;
+ Completer<void> oldCompleter = completer;
+ completer = new Completer();
+ oldCompleter.complete();
+ }),
);
- p.stdout.transform(utf8.decoder).transform(const LineSplitter()).listen((
- String line,
- ) {
- print("stdout> $line");
- });
- p.stderr.transform(utf8.decoder).transform(const LineSplitter()).listen((
- String line,
- ) {
- print("stderr> $line");
- });
- print("Exit code = ${await p.exitCode}");
+ }
+ while (processesRunning > 0) {
+ await completer.future;
}
- // Don't include the not-compiled stuff as we've (mostly) asked the VM to
- // force compile everything and that the remaining stuff is (mostly) mixins
- // and const classes etc that shouldn't (necessarily) be compiled but is
- // potentially covered in other ways.
- await coverageMerger.mergeFromDirUri(
- repoDirUri.resolve(".dart_tool/package_config.json"),
- coverageTmpDir.uri,
- silent: false,
- extraCoverageIgnores: const [],
- extraCoverageBlockIgnores: const [],
+ for (List<String> runThis in runTheseOneAtATime) {
+ print("Starting $runThis");
+ print(
+ "Finished with exit code "
+ "${await _run(silent, runThis, environment)}",
+ );
+ }
+
+ return coverageTmpDir;
+}
+
+Future<int> _run(
+ bool silent,
+ List<String> runThis,
+ Map<String, String> environment,
+) async {
+ Process p = await Process.start(
+ runThis.first,
+ runThis.skip(1).toList(),
+ environment: environment,
);
+ p.stdout.transform(utf8.decoder).transform(const LineSplitter()).listen((
+ String line,
+ ) {
+ if (!silent) print("stdout> $line");
+ });
+ p.stderr.transform(utf8.decoder).transform(const LineSplitter()).listen((
+ String line,
+ ) {
+ print("stderr> $line");
+ });
+ return await p.exitCode;
}
diff --git a/pkg/front_end/test/run_all_coverage_update.dart b/pkg/front_end/test/run_all_coverage_update.dart
new file mode 100644
index 0000000..619665c
--- /dev/null
+++ b/pkg/front_end/test/run_all_coverage_update.dart
@@ -0,0 +1,21 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io' show Directory;
+
+import '../tool/coverage_merger.dart' show mergeFromDirUri;
+import 'run_all_coverage.dart' show runAllCoverageTests;
+
+Future<void> main() async {
+ Directory coverageTmpDir = await runAllCoverageTests(silent: true);
+
+ await mergeFromDirUri(
+ Uri.base.resolve(".dart_tool/package_config.json"),
+ coverageTmpDir.uri,
+ silent: true,
+ extraCoverageIgnores: ["coverage-ignore(suite):"],
+ extraCoverageBlockIgnores: ["coverage-ignore-block(suite):"],
+ addAndRemoveCommentsInFiles: true,
+ );
+}
diff --git a/pkg/front_end/test/weekly_tester.dart b/pkg/front_end/test/weekly_tester.dart
index 22a613f4..5807b70 100644
--- a/pkg/front_end/test/weekly_tester.dart
+++ b/pkg/front_end/test/weekly_tester.dart
@@ -4,7 +4,7 @@
import 'dart:async' show Future;
import 'dart:convert' show LineSplitter, utf8;
-import 'dart:io' show File, Platform, Process;
+import 'dart:io' show File, Platform, Process, ProcessResult;
Future<void> main(List<String> args) async {
// General idea: Launch - in separate processes - whatever we want to run
@@ -109,6 +109,15 @@
}
shouldThrow = true;
}
+
+ // Now run all coverage and print a diff.
+ WrappedProcess coverageProcess = await run([
+ Platform.script.resolve("run_all_coverage_update.dart").toString(),
+ ], "coverage update");
+ await coverageProcess.process.exitCode;
+ ProcessResult coverageDiffResult = Process.runSync("git", ["diff"]);
+ print(coverageDiffResult.stdout);
+
if (shouldThrow) throw "There were failures!";
}
