fix: improve error reporting on GitHub (#2629)
Group success and skip so it's easy to see the failures!
Update skip icon to ⏭️ so they are easier to scan
Fixes https://github.com/dart-lang/test/issues/2628
diff --git a/pkgs/test/CHANGELOG.md b/pkgs/test/CHANGELOG.md
index a2aefbb..8246b5e 100644
--- a/pkgs/test/CHANGELOG.md
+++ b/pkgs/test/CHANGELOG.md
@@ -3,8 +3,12 @@
* Ignore an error locating the SDK directory on platforms where the
`resolvedExecutable` is unexpectedly `null`.
* Fix a bug where `-c exe` tests would hang on exit on windows.
-* Allow `analyzer` version `13.x.x`.
+* `GithubReporter`:
+ * Group contiguous passing and skipped tests into collapsible groups to
+ reduce log noise in GitHub Actions.
+ * Updated skipped icon to ⏭️.
* Increase SDK constraint to ^3.10.0.
+* Allow `analyzer` version `13.x.x`.
## 1.31.0
diff --git a/pkgs/test/test/runner/github_reporter_test.dart b/pkgs/test/test/runner/github_reporter_test.dart
index f52ceb5..faa1ba6 100644
--- a/pkgs/test/test/runner/github_reporter_test.dart
+++ b/pkgs/test/test/runner/github_reporter_test.dart
@@ -30,9 +30,11 @@
test('success 2', () {});
test('success 3', () {});''',
'''
+ ::group::✅ Passing tests
✅ success 1
✅ success 2
✅ success 3
+ ::endgroup::
🎉 3 tests passed.''',
);
});
@@ -42,8 +44,10 @@
'''
test('success 1', () {});''',
[
+ '::group::✅ Passing tests',
'✅ [VM, Kernel] success 1',
'✅ [Chrome, Dart2Js] success 1',
+ '::endgroup::',
'🎉 2 tests passed.',
],
args: ['-p', 'vm,chrome'],
@@ -104,12 +108,16 @@
oh no
test.dart 6:33 main.<fn>
::endgroup::
+ ::group::✅ Passing tests
✅ success 1
+ ::endgroup::
::group::❌ failure 2 (failed)
oh no
test.dart 8:33 main.<fn>
::endgroup::
+ ::group::✅ Passing tests
✅ success 2
+ ::endgroup::
::error::2 tests passed, 2 failed.''',
);
});
@@ -148,7 +156,9 @@
third error
test.dart 12:34 main.<fn>.<fn>
::endgroup::
+ ::group::✅ Passing tests
✅ wait
+ ::endgroup::
::error::1 test passed, 1 failed.''',
);
});
@@ -164,7 +174,9 @@
test('second test so that the first failure is reported', () {});''',
'''
+ ::group::✅ Passing tests
✅ fail after completion
+ ::endgroup::
::group::❌ fail after completion (failed after test completion)
foo
test.dart 8:62 main.<fn>.<fn>
@@ -175,7 +187,9 @@
of pending async work.
test.dart 8:62 main.<fn>.<fn>
::endgroup::
+ ::group::✅ Passing tests
✅ second test so that the first failure is reported
+ ::endgroup::
::error::1 test passed, 1 failed.''',
);
});
@@ -223,12 +237,16 @@
return testDone.future;
});''',
'''
+ ::group::✅ Passing tests
✅ test
+ ::endgroup::
one
two
three
four
+ ::group::✅ Passing tests
✅ wait
+ ::endgroup::
🎉 2 tests passed.''',
);
});
@@ -242,9 +260,11 @@
test('skip 2', () {}, skip: true);
test('skip 3', () {}, skip: true);''',
'''
- ❎ skip 1 (skipped)
- ❎ skip 2 (skipped)
- ❎ skip 3 (skipped)
+ ::group::⏭️ Skipped tests
+ ⏭️ skip 1 (skipped)
+ ⏭️ skip 2 (skipped)
+ ⏭️ skip 3 (skipped)
+ ::endgroup::
🎉 0 tests passed, 3 skipped.''',
);
});
@@ -258,9 +278,11 @@
test('test 3', () {});
}, skip: true);''',
'''
- ❎ skip test 1 (skipped)
- ❎ skip test 2 (skipped)
- ❎ skip test 3 (skipped)
+ ::group::⏭️ Skipped tests
+ ⏭️ skip test 1 (skipped)
+ ⏭️ skip test 2 (skipped)
+ ⏭️ skip test 3 (skipped)
+ ::endgroup::
🎉 0 tests passed, 3 skipped.''',
);
});
@@ -273,10 +295,18 @@
test('skip 2', () {}, skip: true);
test('success 2', () {});''',
'''
- ❎ skip 1 (skipped)
+ ::group::⏭️ Skipped tests
+ ⏭️ skip 1 (skipped)
+ ::endgroup::
+ ::group::✅ Passing tests
✅ success 1
- ❎ skip 2 (skipped)
+ ::endgroup::
+ ::group::⏭️ Skipped tests
+ ⏭️ skip 2 (skipped)
+ ::endgroup::
+ ::group::✅ Passing tests
✅ success 2
+ ::endgroup::
🎉 2 tests passed, 2 skipped.''',
);
});
@@ -295,14 +325,22 @@
oh no
test.dart 6:35 main.<fn>
::endgroup::
- ❎ skip 1 (skipped)
+ ::group::⏭️ Skipped tests
+ ⏭️ skip 1 (skipped)
+ ::endgroup::
+ ::group::✅ Passing tests
✅ success 1
+ ::endgroup::
::group::❌ failure 2 (failed)
oh no
test.dart 9:35 main.<fn>
::endgroup::
- ❎ skip 2 (skipped)
+ ::group::⏭️ Skipped tests
+ ⏭️ skip 2 (skipped)
+ ::endgroup::
+ ::group::✅ Passing tests
✅ success 2
+ ::endgroup::
::error::2 tests passed, 2 failed, 2 skipped.''',
);
});
@@ -313,10 +351,10 @@
test('skip 1', () {}, skip: 'some reason');
test('skip 2', () {}, skip: 'or another');''',
'''
- ::group::❎ skip 1 (skipped)
+ ::group::⏭️ Skipped tests
+ ⏭️ skip 1 (skipped)
Skip: some reason
- ::endgroup::
- ::group::❎ skip 2 (skipped)
+ ⏭️ skip 2 (skipped)
Skip: or another
::endgroup::
🎉 0 tests passed, 2 skipped.''',
@@ -333,7 +371,9 @@
test('test 1', () {});
});''',
'''
+ ::group::✅ Passing tests
✅ one test 1
+ ::endgroup::
🎉 1 test passed.''',
);
});
@@ -350,7 +390,9 @@
::group::✅ one (setUpAll)
one
::endgroup::
+ ::group::✅ Passing tests
✅ one test 1
+ ::endgroup::
::group::✅ one (tearDownAll)
two
::endgroup::
diff --git a/pkgs/test_core/CHANGELOG.md b/pkgs/test_core/CHANGELOG.md
index be69f08..058346f 100644
--- a/pkgs/test_core/CHANGELOG.md
+++ b/pkgs/test_core/CHANGELOG.md
@@ -6,6 +6,10 @@
environment variables for color output detection.
* Fix a bug where `-c exe` tests would hang on exit on windows.
* Update `parse_metadata.dart` to be compatible with `analyzer >=8.0.0 <14.0.0`.
+* `GithubReporter`:
+ * Group contiguous passing and skipped tests into collapsible groups to
+ reduce log noise in GitHub Actions.
+ * Updated skipped icon to ⏭️.
## 0.6.17
diff --git a/pkgs/test_core/lib/src/runner/reporter/github.dart b/pkgs/test_core/lib/src/runner/reporter/github.dart
index 39094b5..48d42db 100644
--- a/pkgs/test_core/lib/src/runner/reporter/github.dart
+++ b/pkgs/test_core/lib/src/runner/reporter/github.dart
@@ -44,6 +44,9 @@
final Set<LiveTest> _completedTests = {};
+ /// The github markdown `::group::` that is currently open.
+ var _activeGroup = _ReportGroup.ungrouped;
+
/// Watches the tests run by [engine] and prints their results as JSON.
static GithubReporter watch(
Engine engine,
@@ -112,6 +115,10 @@
if (_completedTests.contains(liveTest)) {
// The test has already completed and it's previous messages were
// written out; ensure this post-completion output is not lost.
+ if (!_activeGroup.isUngrouped) {
+ _sink.writeln(_GithubMarkup.endGroup);
+ _activeGroup = _ReportGroup.ungrouped;
+ }
_sink.writeln(message.text);
} else {
_testMessages.putIfAbsent(liveTest, () => []).add(message);
@@ -166,9 +173,32 @@
'[${test.suite.platform.runtime.name}, '
'${test.suite.platform.compiler.name}] $name';
}
- if (messages.isEmpty && errors.isEmpty) {
+ if (skipped) {
+ if (_activeGroup.isPassing) {
+ _sink.writeln(_GithubMarkup.endGroup);
+ }
+ if (!_activeGroup.isSkipped) {
+ _sink.writeln(_GithubMarkup.startGroup('⏭️ Skipped tests'));
+ _activeGroup = _ReportGroup.skipped;
+ }
+ _sink.writeln('$prefix $name$statusSuffix');
+ for (var message in messages) {
+ _sink.writeln(message.text);
+ }
+ } else if (messages.isEmpty && errors.isEmpty) {
+ if (_activeGroup.isSkipped) {
+ _sink.writeln(_GithubMarkup.endGroup);
+ }
+ if (!_activeGroup.isPassing) {
+ _sink.writeln(_GithubMarkup.startGroup('✅ Passing tests'));
+ _activeGroup = _ReportGroup.passing;
+ }
_sink.writeln('$prefix $name$statusSuffix');
} else {
+ if (!_activeGroup.isUngrouped) {
+ _sink.writeln(_GithubMarkup.endGroup);
+ _activeGroup = _ReportGroup.ungrouped;
+ }
_sink.writeln(_GithubMarkup.startGroup('$prefix $name$statusSuffix'));
for (var message in messages) {
_sink.writeln(message.text);
@@ -201,6 +231,10 @@
'${test.suite.platform.compiler.name}] $name';
}
+ if (!_activeGroup.isUngrouped) {
+ _sink.writeln(_GithubMarkup.endGroup);
+ _activeGroup = _ReportGroup.ungrouped;
+ }
_sink.writeln(_GithubMarkup.startGroup('$prefix $name$statusSuffix'));
_sink.writeln('$error');
_sink.writeln(stackTrace.toString().trimRight());
@@ -211,6 +245,11 @@
void _onDone(bool? success) {
_cancel();
+ if (!_activeGroup.isUngrouped) {
+ _sink.writeln(_GithubMarkup.endGroup);
+ _activeGroup = _ReportGroup.ungrouped;
+ }
+
_sink.writeln();
final hadFailures = _engine.failed.isNotEmpty;
@@ -236,7 +275,7 @@
abstract class _GithubMarkup {
// Char sets avilable at https://www.compart.com/en/unicode/.
static const String passed = '✅';
- static const String skipped = '❎';
+ static const String skipped = '⏭️';
static const String failed = '❌';
// The 'synthetic' icon is currently not used but is something to consider in
// order to draw a distinction between user tests and test-like supporting
@@ -251,3 +290,13 @@
static String error(String message) => '::error::$message';
}
+
+enum _ReportGroup {
+ passing,
+ skipped,
+ ungrouped;
+
+ bool get isPassing => this == _ReportGroup.passing;
+ bool get isSkipped => this == _ReportGroup.skipped;
+ bool get isUngrouped => this == _ReportGroup.ungrouped;
+}
