feat(compliance): add GitHub Actions Job Summary to test runs (#527)
- Bump vendor/Http11Probe submodule to latest remote commit.
- Update golden reports and summary for inert test removals in Http11Probe.
- Wrap long lines in compliance_test.dart to satisfy 80-column line rule.
diff --git a/pkgs/_shelf_compliance/reports/shelf/Compliance.json b/pkgs/_shelf_compliance/reports/shelf/Compliance.json
index e34b54a..5c84a5b 100644
--- a/pkgs/_shelf_compliance/reports/shelf/Compliance.json
+++ b/pkgs/_shelf_compliance/reports/shelf/Compliance.json
@@ -1,17 +1,5 @@
[
{
- "id": "COMP-405-ALLOW",
- "description": "405 response must include an Allow header",
- "category": "Compliance",
- "rfcReference": "RFC 9110 §15.5.6",
- "rfcLevel": "Must",
- "expected": "405 + Allow header",
- "verdict": "Warn",
- "statusCode": 404,
- "rawRequest": "DELETE / HTTP/1.1\r\nHost: 127.0.0.1:<PORT>\r\n\r\n",
- "rawResponse": "HTTP/1.1 404 Not Found\r\nx-powered-by: Dart with package:shelf\r\ndate: <DATE>\r\ncontent-length: 9\r\nx-frame-options: SAMEORIGIN\r\ncontent-type: text/plain; charset=utf-8\r\nx-xss-protection: 1; mode=block\r\nx-content-type-options: nosniff\r\n\r\nNot Found"
- },
- {
"id": "COMP-ABSOLUTE-FORM",
"description": "Absolute-form request-target — server should accept per RFC",
"category": "Compliance",
@@ -396,18 +384,6 @@
"rawRequest": "POST / HTTP/1.0\r\nHost: 127.0.0.1:<PORT>\r\nExpect: 100-continue\r\nContent-Length: 5\r\n\r\nhello"
},
{
- "id": "COMP-NO-CL-IN-204",
- "description": "Server must not send Content-Length in a 204 response",
- "category": "Compliance",
- "rfcReference": "RFC 9110 §8.6",
- "rfcLevel": "Must",
- "expected": "204 without CL, or 405",
- "verdict": "Warn",
- "statusCode": 404,
- "rawRequest": "OPTIONS / HTTP/1.1\r\nHost: 127.0.0.1:<PORT>\r\n\r\n",
- "rawResponse": "HTTP/1.1 404 Not Found\r\nx-powered-by: Dart with package:shelf\r\ndate: <DATE>\r\ncontent-length: 9\r\nx-frame-options: SAMEORIGIN\r\ncontent-type: text/plain; charset=utf-8\r\nx-xss-protection: 1; mode=block\r\nx-content-type-options: nosniff\r\n\r\nNot Found"
- },
- {
"id": "COMP-OPTIONS-ALLOW",
"description": "OPTIONS response should include Allow header listing supported methods",
"category": "Compliance",
diff --git a/pkgs/_shelf_compliance/reports/shelf/Cookies.json b/pkgs/_shelf_compliance/reports/shelf/Cookies.json
index 277fa8f..7e62b53 100644
--- a/pkgs/_shelf_compliance/reports/shelf/Cookies.json
+++ b/pkgs/_shelf_compliance/reports/shelf/Cookies.json
@@ -68,7 +68,7 @@
"verdict": "Pass",
"statusCode": 200,
"rawRequest": "GET /echo HTTP/1.1\r\nHost: 127.0.0.1:<PORT>\r\nCookie: a=1\r\nCookie: b=2\r\n\r\n",
- "rawResponse": "HTTP/1.1 200 OK\r\nx-powered-by: Dart with package:shelf\r\ndate: <DATE>\r\ncontent-length: 40\r\nx-frame-options: SAMEORIGIN\r\ncontent-type: text/plain; charset=utf-8\r\nx-xss-protection: 1; mode=block\r\nx-content-type-options: nosniff\r\n\r\ncookie: a=1,b=2\r\nhost: 127.0.0.1:<PORT>\r\n",
+ "rawResponse": "HTTP/1.1 200 OK\r\nx-powered-by: Dart with package:shelf\r\ndate: <DATE>\r\ncontent-length: 41\r\nx-frame-options: SAMEORIGIN\r\ncontent-type: text/plain; charset=utf-8\r\nx-xss-protection: 1; mode=block\r\nx-content-type-options: nosniff\r\n\r\ncookie: a=1; b=2\r\nhost: 127.0.0.1:<PORT>\r\n",
"behavioralNote": "Both cookies echoed"
},
{
diff --git a/pkgs/_shelf_compliance/shelf_summary.md b/pkgs/_shelf_compliance/shelf_summary.md
index c5235a9..709d676 100644
--- a/pkgs/_shelf_compliance/shelf_summary.md
+++ b/pkgs/_shelf_compliance/shelf_summary.md
@@ -2,10 +2,10 @@
| Category | Count |
| --- | --- |
-| Total | 215 |
+| Total | 213 |
| Passed | 109 |
| Failed | 49 |
-| Warnings | 57 |
+| Warnings | 55 |
| Errors | 0 |
## Failed or Warning Results
@@ -19,7 +19,6 @@
| CAP-INM-UNQUOTED | Capabilities | Warn | If-None-Match with unquoted ETag |
| CAP-INM-WILDCARD | Capabilities | Warn | If-None-Match: * on existing resource returns 304 |
| CAP-LAST-MODIFIED-304 | Capabilities | Warn | Last-Modified conditional GET returns 304 Not Modified |
-| COMP-405-ALLOW | Compliance | Warn | 405 response must include an Allow header |
| COMP-ACCEPT-NONSENSE | Compliance | Warn | Unrecognized Accept value — server may return 406 or default representation |
| COMP-ASTERISK-WITH-GET | Compliance | Fail | Asterisk-form (*) request-target with GET must be rejected |
| COMP-CHUNKED-NO-FINAL | Compliance | Fail | Chunked body without zero terminator — incomplete transfer |
@@ -36,7 +35,6 @@
| COMP-LEADING-CRLF | Compliance | Warn | Leading CRLF before request-line — server may ignore per RFC |
| COMP-METHOD-CASE | Compliance | Fail | Lowercase method 'get' — methods are case-sensitive per RFC |
| COMP-METHOD-TRACE | Compliance | Fail | TRACE request — should be disabled in production |
-| COMP-NO-CL-IN-204 | Compliance | Warn | Server must not send Content-Length in a 204 response |
| COMP-OPTIONS-ALLOW | Compliance | Fail | OPTIONS response should include Allow header listing supported methods |
| COMP-OPTIONS-STAR | Compliance | Fail | OPTIONS * is the only valid asterisk-form request |
| COMP-POST-CL-UNDERSEND | Compliance | Fail | POST with Content-Length: 10 but only 5 bytes sent — incomplete body |
diff --git a/pkgs/_shelf_compliance/test/compliance_test.dart b/pkgs/_shelf_compliance/test/compliance_test.dart
index c8df3d7..56ada8d 100644
--- a/pkgs/_shelf_compliance/test/compliance_test.dart
+++ b/pkgs/_shelf_compliance/test/compliance_test.dart
@@ -34,7 +34,15 @@
print('::warning file=$filePath,title=$title::$message');
}
+final _improvements = <Map<String, String>>[];
+final _regressions = <Map<String, String>>[];
+final _benignChanges = <Map<String, String>>[];
+var _totalProbeTests = 0;
+var _matchingBaseline = 0;
+
void main() {
+ tearDownAll(_writeGithubStepSummary);
+
test('Verify categories list is complete', () async {
final helpProcess = await TestProcess.start('dotnet', [
'run',
@@ -102,51 +110,161 @@
);
}
- tearDownAll(() async {
- print('Generating combined summary for $name...');
- final reportsDir = Directory(p.join(tempDir.path, 'reports', name));
- final summary = generateSummary(reportsDir);
-
- final goldenSummary = File('${name}_summary.md');
-
- final sanitizedSummary = canonicalize(
- summary,
- 0,
- ); // No port to sanitize in summary usually
-
- if (!goldenSummary.existsSync()) {
- fail(
- 'Golden summary missing! Please run tool/update_goldens.dart to '
- 'create it.',
- );
- }
- final expectedSummary = goldenSummary.readAsStringSync();
- if (sanitizedSummary != expectedSummary) {
- if (hasRegressions) {
- print('MISMATCH in summary!');
- print('Generated summary in temp dir.');
- print('Golden summary: ${goldenSummary.path}');
- fail(
- 'Generated summary does not match golden and there were '
- 'regressions.',
- );
- } else {
- _printGithubWarning(
- 'pkgs/_shelf_compliance/${name}_summary.md',
- 'Compliance Summary Improved!',
- 'The summary improved or changed benignly but does not match the '
- 'golden. Run tool/update_goldens.dart to tighten.',
- );
- }
- }
-
- // Clean up temp directory
+ // Note: In package:test, tearDownAll callbacks execute in reverse
+ // registration order (LIFO). We register cleanup first so that summary
+ // verification runs before temp directory deletion.
+ tearDownAll(() {
print('Cleaning up temp directory: ${tempDir.path}');
tempDir.deleteSync(recursive: true);
});
+
+ tearDownAll(() {
+ _verifySummary(
+ name: name,
+ tempDir: tempDir,
+ hasRegressions: hasRegressions,
+ );
+ });
});
}
+void _verifySummary({
+ required String name,
+ required Directory tempDir,
+ required bool hasRegressions,
+}) {
+ print('Generating combined summary for $name...');
+ final reportsDir = Directory(p.join(tempDir.path, 'reports', name));
+ final summary = generateSummary(reportsDir);
+
+ final goldenSummary = File('${name}_summary.md');
+
+ final sanitizedSummary = canonicalize(
+ summary,
+ 0,
+ ); // No port to sanitize in summary usually
+
+ if (!goldenSummary.existsSync()) {
+ fail(
+ 'Golden summary missing! Please run tool/update_goldens.dart to '
+ 'create it.',
+ );
+ }
+ final expectedSummary = goldenSummary.readAsStringSync();
+ if (sanitizedSummary != expectedSummary) {
+ if (hasRegressions) {
+ print('MISMATCH in summary!');
+ print('Generated summary in temp dir.');
+ print('Golden summary: ${goldenSummary.path}');
+ fail(
+ 'Generated summary does not match golden and there were '
+ 'regressions.',
+ );
+ } else {
+ _printGithubWarning(
+ 'pkgs/_shelf_compliance/${name}_summary.md',
+ 'Compliance Summary Improved!',
+ 'The summary improved or changed benignly but does not match the '
+ 'golden. Run tool/update_goldens.dart to tighten.',
+ );
+ }
+ }
+}
+
+void _writeGithubStepSummary() {
+ final stepSummaryPath = Platform.environment['GITHUB_STEP_SUMMARY'];
+ if (stepSummaryPath == null) return;
+
+ final file = File(stepSummaryPath);
+ final buffer = StringBuffer();
+ buffer.writeln('## 🛡️ HTTP/1.1 Compliance Test Summary');
+ buffer.writeln();
+
+ if (_regressions.isNotEmpty) {
+ buffer.writeln('### ❌ Regressions Detected');
+ buffer.writeln(
+ 'The following tests regressed compared to the baseline. '
+ 'The build has been marked as failed.',
+ );
+ buffer.writeln();
+ buffer.writeln(
+ '| Test ID | Category | Baseline Verdict | Actual Verdict |',
+ );
+ buffer.writeln('| --- | --- | --- | --- |');
+ for (var r in _regressions) {
+ buffer.writeln(
+ '| `${r['id']}` | ${r['category']} | '
+ '**${r['expected']}** | **${r['actual']}** |',
+ );
+ }
+ buffer.writeln();
+ }
+
+ if (_improvements.isNotEmpty) {
+ buffer.writeln('### 🚀 Improvements Detected');
+ buffer.writeln(
+ 'The following tests improved compared to the baseline! '
+ 'Please run `dart run tool/update_goldens.dart` in '
+ '`pkgs/_shelf_compliance` to update the goldens.',
+ );
+ buffer.writeln();
+ buffer.writeln(
+ '| Test ID | Category | Baseline Verdict | Actual Verdict |',
+ );
+ buffer.writeln('| --- | --- | --- | --- |');
+ for (var imp in _improvements) {
+ buffer.writeln(
+ '| `${imp['id']}` | ${imp['category']} | '
+ '**${imp['expected']}** | **${imp['actual']}** |',
+ );
+ }
+ buffer.writeln();
+ }
+
+ if (_benignChanges.isNotEmpty) {
+ buffer.writeln('### ⚠️ Benign Changes Detected');
+ buffer.writeln(
+ 'The following tests had benign changes (verdicts remain '
+ 'unchanged). Please run `dart run tool/update_goldens.dart` '
+ 'to update.',
+ );
+ buffer.writeln();
+ buffer.writeln('| Test ID | Category | Verdict |');
+ buffer.writeln('| --- | --- | --- |');
+ for (var bc in _benignChanges) {
+ buffer.writeln(
+ '| `${bc['id']}` | ${bc['category']} | **${bc['verdict']}** |',
+ );
+ }
+ buffer.writeln();
+ }
+
+ if (_regressions.isEmpty && _improvements.isEmpty && _benignChanges.isEmpty) {
+ buffer.writeln(
+ '> 🎉 **All $_totalProbeTests compliance tests match the '
+ 'baseline perfectly!** No changes or regressions detected '
+ 'compared to the goldens.',
+ );
+ buffer.writeln();
+ } else {
+ buffer.writeln('### 📊 Stats Overview');
+ buffer.writeln('* **Total tests compared**: $_totalProbeTests');
+ buffer.writeln('* ✅ **Matches baseline**: $_matchingBaseline');
+ if (_improvements.isNotEmpty) {
+ buffer.writeln('* 🚀 **Improved**: ${_improvements.length}');
+ }
+ if (_benignChanges.isNotEmpty) {
+ buffer.writeln('* ⚠️ **Benign changes**: ${_benignChanges.length}');
+ }
+ if (_regressions.isNotEmpty) {
+ buffer.writeln('* ❌ **Regressions**: ${_regressions.length}');
+ }
+ buffer.writeln();
+ }
+
+ file.writeAsStringSync(buffer.toString(), mode: FileMode.append);
+}
+
void _testCompliance({
required String name,
required String serverPath,
@@ -178,8 +296,8 @@
if (!goldenReport.existsSync()) {
fail(
- 'Golden report missing for $category! Please run tool/update_goldens.dart '
- 'to create it.',
+ 'Golden report missing for $category! Please run '
+ 'tool/update_goldens.dart to create it.',
);
}
@@ -200,9 +318,11 @@
reason: 'Length of results changed',
);
+ final failures = <String>[];
for (var i = 0; i < filteredMaps.length; i++) {
final actual = filteredMaps[i];
final expected = expectedResults[i];
+ _totalProbeTests++;
expect(actual['id'], equals(expected['id']));
@@ -214,15 +334,23 @@
if (actualRank < expectedRank) {
reportRegression();
-
- expect(
- actual,
- equals(expected),
- reason:
- 'Test ${actual['id']} regressed from $expectedVerdictStr to '
- '$actualVerdictStr',
+ _regressions.add({
+ 'id': actual['id'] as String,
+ 'category': category,
+ 'expected': expectedVerdictStr,
+ 'actual': actualVerdictStr,
+ });
+ failures.add(
+ 'Test ${actual['id']} regressed from $expectedVerdictStr to '
+ '$actualVerdictStr',
);
} else if (actualRank > expectedRank) {
+ _improvements.add({
+ 'id': actual['id'] as String,
+ 'category': category,
+ 'expected': expectedVerdictStr,
+ 'actual': actualVerdictStr,
+ });
_printGithubWarning(
'pkgs/_shelf_compliance/reports/$name/$category.json',
'Compliance Test Improved!',
@@ -231,14 +359,25 @@
);
} else {
if (!const DeepCollectionEquality().equals(actual, expected)) {
+ _benignChanges.add({
+ 'id': actual['id'] as String,
+ 'category': category,
+ 'verdict': actualVerdictStr,
+ });
_printGithubWarning(
'pkgs/_shelf_compliance/reports/$name/$category.json',
'Compliance Test Changed!',
'Test ${actual['id']} changed benignly (verdict remains '
'$actualVerdictStr). Run tool/update_goldens.dart to tighten.',
);
+ } else {
+ _matchingBaseline++;
}
}
}
+
+ if (failures.isNotEmpty) {
+ fail(failures.join('\n'));
+ }
});
}
diff --git a/vendor/Http11Probe b/vendor/Http11Probe
index 4ebf9ea..3876ad7 160000
--- a/vendor/Http11Probe
+++ b/vendor/Http11Probe
@@ -1 +1 @@
-Subproject commit 4ebf9ea2c04b87067520a5a53c91593ec15e9930
+Subproject commit 3876ad71acaf46930677d5167c5f1358b514fccd
