Merge pull request dart-lang/markdown#446 from jonasfj/crashtest

Wrote some crashtest for running against markdown files from pub.dev

pkgs/markdown/dart_test.yaml

diff --git a/pkgs/markdown/dart_test.yaml b/pkgs/markdown/dart_test.yaml
new file mode 100644
index 0000000..588b409
--- /dev/null
+++ b/pkgs/markdown/dart_test.yaml
@@ -0,0 +1,6 @@
+tags:
+  crash_test:
+    skip: 'Only run crash_test tests manually with `dart test -P crash_test`'
+    presets:
+      crash_test:
+        skip: false # Don't skip when running in -P crash_test

pkgs/markdown/pubspec.yaml

diff --git a/pkgs/markdown/pubspec.yaml b/pkgs/markdown/pubspec.yaml
index b44329b..9145fe4 100644
--- a/pkgs/markdown/pubspec.yaml
+++ b/pkgs/markdown/pubspec.yaml
@@ -21,9 +21,12 @@
   build_web_compilers: ^3.0.0
   collection: ^1.15.0
   html: ^0.15.0
+  http: ^0.13.5
   io: ^1.0.0
   js: ^0.6.3
   lints: ^2.0.0
   path: ^1.8.0
+  pool: ^1.5.1
+  tar: ^0.5.5+1
   test: ^1.16.0
   yaml: ^3.0.0

pkgs/markdown/test/crash_test.dart

diff --git a/pkgs/markdown/test/crash_test.dart b/pkgs/markdown/test/crash_test.dart
new file mode 100644
index 0000000..3d06683
--- /dev/null
+++ b/pkgs/markdown/test/crash_test.dart
@@ -0,0 +1,128 @@
+// Copyright (c) 2022, the Dart project authors.  Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:http/http.dart' as http;
+import 'package:http/retry.dart' as http;
+import 'package:markdown/markdown.dart';
+import 'package:pool/pool.dart';
+import 'package:tar/tar.dart';
+import 'package:test/test.dart';
+
+// ignore_for_file: avoid_dynamic_calls
+
+void main() async {
+  // This test is a really dumb and very slow crash-test.
+  // It downloads the latest package version for each package on pub.dev
+  // and tries to parse all `*.md` files in the package, counting the number
+  // of times where the parser throws.
+  //
+  // Needless to say, this test is very slow and running it eats a lot of CPU.
+  // But it's a fairly good way to try a lot of real-world markdown text to see
+  // if any of the poorly formatted markdown causes the parser to crash.
+  test(
+    'crash test',
+    () async {
+      final c = http.RetryClient(http.Client());
+      Future<dynamic> getJson(String url) async {
+        final u = Uri.tryParse(url);
+        if (u == null) {
+          return null;
+        }
+        try {
+          final data = await c.read(u);
+          try {
+            return jsonDecode(data);
+          } on FormatException {
+            return null;
+          }
+        } on http.ClientException {
+          return null;
+        } on IOException {
+          return null;
+        }
+      }
+
+      final packages =
+          ((await getJson('https://pub.dev/api/package-names'))['packages']
+                  as List)
+              .cast<String>();
+      print('Found ${packages.length} packages to scan');
+
+      final errors = <String>[];
+      final pool = Pool(50);
+      var count = 0;
+      var skipped = 0;
+      var lastStatus = DateTime.now();
+      await Future.wait(packages.map((package) async {
+        await pool.withResource(() async {
+          final versionsResponse =
+              await getJson('https://pub.dev/api/packages/$package');
+          final archiveUrl = Uri.tryParse(
+            versionsResponse['latest']?['archive_url'] as String? ?? '',
+          );
+          if (archiveUrl == null) {
+            skipped++;
+            return;
+          }
+          late List<int> archive;
+          try {
+            archive = gzip.decode(await c.readBytes(archiveUrl));
+          } on http.ClientException {
+            skipped++;
+            return;
+          } on IOException {
+            skipped++;
+            return;
+          }
+          try {
+            await TarReader.forEach(Stream.value(archive), (entry) async {
+              if (entry.name.endsWith('.md')) {
+                late String contents;
+                try {
+                  final bytes = await http.ByteStream(entry.contents).toBytes();
+                  contents = utf8.decode(bytes);
+                } on FormatException {
+                  return; // ignore invalid utf8
+                }
+                try {
+                  markdownToHtml(
+                    contents,
+                    extensionSet: ExtensionSet.gitHubWeb,
+                  );
+                } catch (err, st) {
+                  errors
+                      .add('package:$package/${entry.name}, throws: $err\n$st');
+                }
+              }
+            });
+          } on FormatException {
+            skipped++;
+            return;
+          }
+        });
+        count++;
+        if (DateTime.now().difference(lastStatus) >
+            const Duration(seconds: 30)) {
+          lastStatus = DateTime.now();
+          print('Scanned $count / ${packages.length} (skipped $skipped),'
+              ' found ${errors.length} issues');
+        }
+      }));
+
+      await pool.close();
+      c.close();
+
+      if (errors.isNotEmpty) {
+        print('Found issues:');
+        errors.forEach(print);
+        fail('Found ${errors.length} cases where markdownToHtml threw!');
+      }
+    },
+    timeout: const Timeout(Duration(hours: 1)),
+    tags: 'crash_test', // skipped by default, see: dart_test.yaml
+  );
+}