Run a periodic crash-test (dart-lang/markdown#590)
* Run a periodic crash-test
* Better documentation
diff --git a/pkgs/markdown/.github/workflows/crash_test.yaml b/pkgs/markdown/.github/workflows/crash_test.yaml
new file mode 100644
index 0000000..fcd1afa
--- /dev/null
+++ b/pkgs/markdown/.github/workflows/crash_test.yaml
@@ -0,0 +1,20 @@
+# Run against all markdown files in latest version of packages on pub.dev to
+# see if any can provoke a crash
+
+name: Crash Tests
+
+on:
+ schedule:
+ # “At 00:00 (UTC) on Sunday.”
+ - cron: '0 0 * * 0'
+
+jobs:
+ crash-test:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11
+ - uses: dart-lang/setup-dart@fedb1266e91cf51be2fdb382869461a434b920a3
+ - name: Install dependencies
+ run: dart pub get
+ - name: Run crash_test.dart
+ run: dart test -P crash_test test/crash_test.dart
diff --git a/pkgs/markdown/test/crash_test.dart b/pkgs/markdown/test/crash_test.dart
index 3d06683..a6f427d 100644
--- a/pkgs/markdown/test/crash_test.dart
+++ b/pkgs/markdown/test/crash_test.dart
@@ -4,6 +4,7 @@
import 'dart:convert';
import 'dart:io';
+import 'dart:isolate';
import 'package:http/http.dart' as http;
import 'package:http/retry.dart' as http;
@@ -14,6 +15,18 @@
// ignore_for_file: avoid_dynamic_calls
+const extensions = [
+ '.md',
+ '.mkd',
+ '.mdwn',
+ '.mdown',
+ '.mdtxt',
+ '.mdtext',
+ '.markdown',
+ 'README',
+ 'CHANGELOG',
+];
+
void main() async {
// This test is a really dumb and very slow crash-test.
// It downloads the latest package version for each package on pub.dev
@@ -26,6 +39,16 @@
test(
'crash test',
() async {
+ final started = DateTime.now();
+ var lastStatus = DateTime(0);
+ void status(String Function() message) {
+ if (DateTime.now().difference(lastStatus) >
+ const Duration(seconds: 30)) {
+ lastStatus = DateTime.now();
+ print(message());
+ }
+ }
+
final c = http.RetryClient(http.Client());
Future<dynamic> getJson(String url) async {
final u = Uri.tryParse(url);
@@ -50,27 +73,47 @@
((await getJson('https://pub.dev/api/package-names'))['packages']
as List)
.cast<String>();
- print('Found ${packages.length} packages to scan');
+ //.take(3).toList(); // useful when testing
+ print('## Found ${packages.length} packages to scan');
- final errors = <String>[];
- final pool = Pool(50);
var count = 0;
- var skipped = 0;
- var lastStatus = DateTime.now();
+ final pool = Pool(50);
+ final packageVersions = <PackageVersion>[];
await Future.wait(packages.map((package) async {
await pool.withResource(() async {
- final versionsResponse =
- await getJson('https://pub.dev/api/packages/$package');
- final archiveUrl = Uri.tryParse(
- versionsResponse['latest']?['archive_url'] as String? ?? '',
+ final response = await getJson(
+ 'https://pub.dev/api/packages/$package',
);
+ final entry = response['latest'] as Map?;
+ if (entry != null) {
+ packageVersions.add(PackageVersion(
+ package: package,
+ version: entry['version'] as String,
+ archiveUrl: entry['archive_url'] as String,
+ ));
+ }
+ count++;
+ status(
+ () => 'Listed versions for $count / ${packages.length} packages',
+ );
+ });
+ }));
+
+ print('## Found ${packageVersions.length} package versions to scan');
+
+ count = 0;
+ final errors = <String>[];
+ var skipped = 0;
+ await Future.wait(packageVersions.map((pv) async {
+ await pool.withResource(() async {
+ final archiveUrl = Uri.tryParse(pv.archiveUrl);
if (archiveUrl == null) {
skipped++;
return;
}
late List<int> archive;
try {
- archive = gzip.decode(await c.readBytes(archiveUrl));
+ archive = await c.readBytes(archiveUrl);
} on http.ClientException {
skipped++;
return;
@@ -78,51 +121,106 @@
skipped++;
return;
}
- try {
- await TarReader.forEach(Stream.value(archive), (entry) async {
- if (entry.name.endsWith('.md')) {
- late String contents;
- try {
- final bytes = await http.ByteStream(entry.contents).toBytes();
- contents = utf8.decode(bytes);
- } on FormatException {
- return; // ignore invalid utf8
- }
- try {
- markdownToHtml(
- contents,
- extensionSet: ExtensionSet.gitHubWeb,
- );
- } catch (err, st) {
- errors
- .add('package:$package/${entry.name}, throws: $err\n$st');
- }
- }
- });
- } on FormatException {
+
+ final result = await _findMarkdownIssues(
+ pv.package,
+ pv.version,
+ archive,
+ );
+
+ // If tar decoding fails.
+ if (result == null) {
skipped++;
return;
}
+
+ errors.addAll(result);
+ result.forEach(print);
});
count++;
- if (DateTime.now().difference(lastStatus) >
- const Duration(seconds: 30)) {
- lastStatus = DateTime.now();
- print('Scanned $count / ${packages.length} (skipped $skipped),'
- ' found ${errors.length} issues');
- }
+ status(() =>
+ 'Scanned $count / ${packageVersions.length} (skipped $skipped),'
+ ' found ${errors.length} issues');
}));
await pool.close();
c.close();
+ print('## Finished scanning');
+ print('Scanned ${packageVersions.length} package versions in '
+ '${DateTime.now().difference(started)}');
+
if (errors.isNotEmpty) {
print('Found issues:');
errors.forEach(print);
fail('Found ${errors.length} cases where markdownToHtml threw!');
}
},
- timeout: const Timeout(Duration(hours: 1)),
+ timeout: const Timeout(Duration(hours: 5)),
tags: 'crash_test', // skipped by default, see: dart_test.yaml
);
}
+
+class PackageVersion {
+ final String package;
+ final String version;
+ final String archiveUrl;
+
+ PackageVersion({
+ required this.package,
+ required this.version,
+ required this.archiveUrl,
+ });
+}
+
+/// Scans [gzippedArchive] for markdown files and tries to parse them all.
+///
+/// Creates a list of issues that arose when parsing markdown files. The
+/// [package] and [version] strings are used to construct nice issues.
+/// An issue string may be multi-line, but should be printable.
+///
+/// Returns a list of issues, or `null` if decoding and parsing [gzippedArchive]
+/// failed.
+Future<List<String>?> _findMarkdownIssues(
+ String package,
+ String version,
+ List<int> gzippedArchive,
+) async {
+ return Isolate.run<List<String>?>(() async {
+ try {
+ final archive = gzip.decode(gzippedArchive);
+ final issues = <String>[];
+ await TarReader.forEach(Stream.value(archive), (entry) async {
+ if (extensions.any((ext) => entry.name.endsWith(ext))) {
+ late String contents;
+ try {
+ final bytes = await http.ByteStream(entry.contents).toBytes();
+ contents = utf8.decode(bytes);
+ } on FormatException {
+ return; // ignore invalid utf8
+ }
+ final start = DateTime.now();
+ try {
+ markdownToHtml(
+ contents,
+ extensionSet: ExtensionSet.gitHubWeb,
+ );
+ } catch (err, st) {
+ issues.add(
+ 'package:$package-$version/${entry.name}, throws: $err\n$st');
+ }
+ final time = DateTime.now().difference(start);
+ if (time.inSeconds > 30) {
+ issues.add(
+ 'package:$package-$version/${entry.name} took $time to process');
+ }
+ }
+ });
+ return issues;
+ } on FormatException {
+ return null;
+ }
+ }).timeout(const Duration(minutes: 2), onTimeout: () {
+ return ['package:$package-$version failed to be processed in 2 minutes'];
+ });
+}
