Add repro instructions to coverage test output. Also update the coverage test so that it prints detailed results to standard output, so that the repro instructions will produce a useful result. In my experiments, I've found that the number supplied for the `--tasks` argument can have a small effect on coverage numbers (though I'm not sure why), so I've included it in the repro instructions. Change-Id: Ic347a9e5e3c2da0f73ceefbaa9322a60295735d6 Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/368920 Reviewed-by: Jens Johansen <jensj@google.com> Commit-Queue: Paul Berry <paulberry@google.com> Reviewed-by: Johnni Winther <johnniwinther@google.com>
diff --git a/pkg/front_end/test/coverage_suite.dart b/pkg/front_end/test/coverage_suite.dart index f36da0c..db241fa8 100644 --- a/pkg/front_end/test/coverage_suite.dart +++ b/pkg/front_end/test/coverage_suite.dart
@@ -381,9 +381,12 @@ })); } - if (options.verbose) { + if (options.verbose || log != null) { String result = pass ? "PASS" : "FAIL"; print("${testName}: ${result}"); + if (log != null) { + print(" ${log.replaceAll('\n', '\n ')}"); + } } } @@ -393,6 +396,7 @@ // TODO(jensj): More info here would be good. addResult(coverageEntry.key.toString(), false, log: "Error"); } else { + StringBuffer sb = new StringBuffer(); int hitCount = coverageEntry.value.hitCount; int missCount = coverageEntry.value.missCount; double percent = (hitCount / (hitCount + missCount) * 100); @@ -402,13 +406,24 @@ int requireAtLeast = (_expect[coverageEntry.key.toString()] ?? 0.0).floor(); bool pass = percent >= requireAtLeast; - String? log; if (!pass) { - log = "${coverageEntry.value.visualization}\n\n" - "Expected at least $requireAtLeast%, got $percent% " - "($hitCount hits and $missCount misses)."; + sb.write("${coverageEntry.value.visualization}"); + sb.write("\n\nExpected at least $requireAtLeast%, got $percent% " + "($hitCount hits and $missCount misses)."); + sb.write("\n\nTo re-run this test, run:"); + var extraFlags = _assertsEnabled ? ' --enable-asserts' : ''; + // It looks like coverage results vary slightly based on the number of + // tasks, so include a `--tasks=` argument in the repro instructions. + // + // TODO(paulberry): why do coverage results vary based on the number of + // tasks? (Note: possibly due to + // https://github.com/dart-lang/sdk/issues/42061) + sb.write( + "\n\n dart$extraFlags pkg/front_end/test/coverage_suite.dart " + "--tasks=${options.numberOfWorkers}"); } - addResult(coverageEntry.key.toString(), pass, log: log); + addResult(coverageEntry.key.toString(), pass, + log: sb.isEmpty ? null : sb.toString()); } } @@ -507,3 +522,9 @@ ); } } + +final bool _assertsEnabled = () { + bool assertsEnabled = false; + assert(assertsEnabled = true); + return assertsEnabled; +}();