Improve FencedCodeBlockSyntax (dart-lang/markdown#478)
* Improve FencedCodeBlockSyntax
* Update gfm_stats.txt
* Fix some requests
* run tool/stats.dart --update-files
diff --git a/pkgs/markdown/lib/src/block_syntaxes/fenced_code_block_syntax.dart b/pkgs/markdown/lib/src/block_syntaxes/fenced_code_block_syntax.dart
index 0b55f4a..f9713ef 100644
--- a/pkgs/markdown/lib/src/block_syntaxes/fenced_code_block_syntax.dart
+++ b/pkgs/markdown/lib/src/block_syntaxes/fenced_code_block_syntax.dart
@@ -4,7 +4,6 @@
import '../ast.dart';
import '../block_parser.dart';
-import '../charcode.dart';
import '../patterns.dart';
import '../util.dart';
import 'block_syntax.dart';
@@ -19,30 +18,62 @@
const FencedCodeBlockSyntax();
@override
- bool canParse(BlockParser parser) {
- final match = pattern.firstMatch(parser.current);
- if (match == null) return false;
- final codeFence = match.group(1)!;
- final infoString = match.group(2);
- // From the CommonMark spec:
- //
- // > If the info string comes after a backtick fence, it may not contain
- // > any backtick characters.
- return codeFence.codeUnitAt(0) != $backquote ||
- !infoString!.codeUnits.contains($backquote);
+ Node parse(BlockParser parser) {
+ final openingFence = _FenceMatch.fromMatch(pattern.firstMatch(
+ escapePunctuation(parser.current),
+ )!);
+
+ var text = parseChildLines(
+ parser,
+ openingFence.marker,
+ openingFence.indent,
+ ).join('\n');
+
+ if (parser.document.encodeHtml) {
+ text = escapeHtml(text);
+ }
+ if (text.isNotEmpty) {
+ text = '$text\n';
+ }
+
+ final code = Element.text('code', text);
+ if (openingFence.hasLanguage) {
+ var language = decodeHtmlCharacters(openingFence.language);
+ if (parser.document.encodeHtml) {
+ language = escapeHtmlAttribute(language);
+ }
+ code.attributes['class'] = 'language-$language';
+ }
+
+ return Element('pre', [code]);
+ }
+
+ String _removeIndentation(String content, int length) {
+ final text = content.replaceFirst(RegExp('^\\s{0,$length}'), '');
+ return content.substring(content.length - text.length);
}
@override
- List<String> parseChildLines(BlockParser parser, [String? endBlock]) {
- endBlock ??= '';
-
+ List<String> parseChildLines(
+ BlockParser parser, [
+ String openingMarker = '',
+ int indent = 0,
+ ]) {
final childLines = <String>[];
+
parser.advance();
+ _FenceMatch? closingFence;
while (!parser.isDone) {
final match = pattern.firstMatch(parser.current);
- if (match == null || !match[1]!.startsWith(endBlock)) {
- childLines.add(parser.current);
+ closingFence = match == null ? null : _FenceMatch.fromMatch(match);
+
+ // Closing code fences cannot have info strings:
+ // https://spec.commonmark.org/0.30/#example-147
+ if (closingFence == null ||
+ !closingFence.marker.startsWith(openingMarker) ||
+ closingFence.hasInfo) {
+ childLines.add(_removeIndentation(parser.current, indent));
parser.advance();
} else {
parser.advance();
@@ -50,46 +81,55 @@
}
}
+ // https://spec.commonmark.org/0.30/#example-127
+ // https://spec.commonmark.org/0.30/#example-128
+ if (closingFence == null && childLines.last.trim().isEmpty) {
+ childLines.removeLast();
+ }
+
return childLines;
}
+}
- @override
- Node parse(BlockParser parser) {
- // Get the syntax identifier, if there is one.
- final match = pattern.firstMatch(parser.current)!;
- final endBlock = match.group(1);
- var infoString = match.group(2)!;
+class _FenceMatch {
+ _FenceMatch._({
+ required this.indent,
+ required this.marker,
+ required this.info,
+ });
- final childLines = parseChildLines(parser, endBlock);
+ factory _FenceMatch.fromMatch(RegExpMatch match) {
+ String marker;
+ String info;
- // The Markdown tests expect a trailing newline.
- childLines.add('');
-
- var text = childLines.join('\n');
- if (parser.document.encodeHtml) {
- text = escapeHtml(text);
- }
- final code = Element.text('code', text);
-
- // the info-string should be trimmed
- // http://spec.commonmark.org/0.22/#example-100
- infoString = infoString.trim();
- if (infoString.isNotEmpty) {
- // only use the first word in the syntax
- // http://spec.commonmark.org/0.22/#example-100
- final firstSpace = infoString.indexOf(' ');
- if (firstSpace >= 0) {
- infoString = infoString.substring(0, firstSpace);
- }
- infoString = decodeHtmlCharacters(infoString);
- if (parser.document.encodeHtml) {
- infoString = escapeHtmlAttribute(infoString);
- }
- code.attributes['class'] = 'language-$infoString';
+ if (match.namedGroup('backtick') != null) {
+ marker = match.namedGroup('backtick')!;
+ info = match.namedGroup('backtickInfo')!;
+ } else {
+ marker = match.namedGroup('tilde')!;
+ info = match.namedGroup('tildeInfo')!;
}
- final element = Element('pre', [code]);
-
- return element;
+ return _FenceMatch._(
+ indent: match[1]!.length,
+ marker: marker,
+ info: info.trim(),
+ );
}
+
+ final int indent;
+ final String marker;
+
+ // The info-string should be trimmed,
+ // https://spec.commonmark.org/0.30/#info-string.
+ final String info;
+
+ // The first word of the info string is typically used to specify the language
+ // of the code sample,
+ // https://spec.commonmark.org/0.30/#example-143.
+ String get language => info.split(' ').first;
+
+ bool get hasInfo => info.isNotEmpty;
+
+ bool get hasLanguage => language.isNotEmpty;
}
diff --git a/pkgs/markdown/lib/src/patterns.dart b/pkgs/markdown/lib/src/patterns.dart
index c13600d..86531ba 100644
--- a/pkgs/markdown/lib/src/patterns.dart
+++ b/pkgs/markdown/lib/src/patterns.dart
@@ -21,7 +21,9 @@
final indentPattern = RegExp(r'^(?: | {0,3}\t)(.*)$');
/// Fenced code block.
-final codeFencePattern = RegExp(r'^[ ]{0,3}(`{3,}|~{3,})(.*)$');
+final codeFencePattern = RegExp(
+ r'^([ ]{0,3})(?:(?<backtick>`{3,})(?<backtickInfo>[^`]*)|(?<tilde>~{3,})(?<tildeInfo>.*))$',
+);
/// Fenced blockquotes.
final blockquoteFencePattern = RegExp(r'^>{3}\s*$');
@@ -199,6 +201,10 @@
'(?<condition_7>(?:$namedTagDefinition)\\s*\$))',
caseSensitive: false);
+/// ASCII punctuation characters.
+// see https://spec.commonmark.org/0.30/#unicode-whitespace-character.
+const asciiPunctuationCharacters = r'''!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~''';
+
/// A pattern to match HTML entity references and numeric character references.
// https://spec.commonmark.org/0.30/#entity-and-numeric-character-references
final htmlCharactersPattern = RegExp(
diff --git a/pkgs/markdown/lib/src/util.dart b/pkgs/markdown/lib/src/util.dart
index 05ec697..a4861e2 100644
--- a/pkgs/markdown/lib/src/util.dart
+++ b/pkgs/markdown/lib/src/util.dart
@@ -147,3 +147,20 @@
/// Returns the whole match String
String get match => this[0]!;
}
+
+/// Escapes the ASCII punctuation characters after backslash(`\`).
+String escapePunctuation(String input) {
+ final buffer = StringBuffer();
+
+ for (var i = 0; i < input.length; i++) {
+ if (input.codeUnitAt(i) == $backslash) {
+ final next = i + 1 < input.length ? input[i + 1] : null;
+ if (next != null && asciiPunctuationCharacters.contains(next)) {
+ i++;
+ }
+ }
+ buffer.write(input[i]);
+ }
+
+ return buffer.toString();
+}
diff --git a/pkgs/markdown/test/common_mark/backslash_escapes.unit b/pkgs/markdown/test/common_mark/backslash_escapes.unit
index 9b794df..7042aa8 100644
--- a/pkgs/markdown/test/common_mark/backslash_escapes.unit
+++ b/pkgs/markdown/test/common_mark/backslash_escapes.unit
@@ -75,5 +75,5 @@
foo
```
<<<
-<pre><code class="language-foo\+bar">foo
+<pre><code class="language-foo+bar">foo
</code></pre>
diff --git a/pkgs/markdown/test/common_mark/block_quotes.unit b/pkgs/markdown/test/common_mark/block_quotes.unit
index 84559d7..f223442 100644
--- a/pkgs/markdown/test/common_mark/block_quotes.unit
+++ b/pkgs/markdown/test/common_mark/block_quotes.unit
@@ -96,8 +96,7 @@
<pre><code>foo
</code></pre>
</blockquote>
-<pre><code>
-</code></pre>
+<pre><code></code></pre>
>>> Block quotes - 238
> foo
- bar
diff --git a/pkgs/markdown/test/common_mark/fenced_code_blocks.unit b/pkgs/markdown/test/common_mark/fenced_code_blocks.unit
index af54076..6148ae3 100644
--- a/pkgs/markdown/test/common_mark/fenced_code_blocks.unit
+++ b/pkgs/markdown/test/common_mark/fenced_code_blocks.unit
@@ -61,8 +61,7 @@
>>> Fenced code blocks - 126
```
<<<
-<pre><code>
-</code></pre>
+<pre><code></code></pre>
>>> Fenced code blocks - 127
`````
@@ -72,7 +71,6 @@
<pre><code>
```
aaa
-
</code></pre>
>>> Fenced code blocks - 128
> ```
@@ -105,7 +103,7 @@
aaa
```
<<<
-<pre><code> aaa
+<pre><code>aaa
aaa
</code></pre>
>>> Fenced code blocks - 132
@@ -116,7 +114,7 @@
```
<<<
<pre><code>aaa
- aaa
+aaa
aaa
</code></pre>
>>> Fenced code blocks - 133
@@ -126,9 +124,9 @@
aaa
```
<<<
-<pre><code> aaa
- aaa
- aaa
+<pre><code>aaa
+ aaa
+aaa
</code></pre>
>>> Fenced code blocks - 134
```
@@ -160,7 +158,6 @@
<<<
<pre><code>aaa
```
-
</code></pre>
>>> Fenced code blocks - 138
``` ```
@@ -175,7 +172,6 @@
<<<
<pre><code>aaa
~~~ ~~
-
</code></pre>
>>> Fenced code blocks - 140
foo
@@ -245,6 +241,5 @@
``` aaa
```
<<<
-<pre><code></code></pre>
-<pre><code>
+<pre><code>``` aaa
</code></pre>
diff --git a/pkgs/markdown/test/common_mark/lists.unit b/pkgs/markdown/test/common_mark/lists.unit
index e105c3b..7eb42df 100644
--- a/pkgs/markdown/test/common_mark/lists.unit
+++ b/pkgs/markdown/test/common_mark/lists.unit
@@ -279,7 +279,6 @@
</li>
</ul>
<pre><code>- c
-
</code></pre>
>>> Lists - 319
- a
diff --git a/pkgs/markdown/test/gfm/backslash_escapes.unit b/pkgs/markdown/test/gfm/backslash_escapes.unit
index e0f5dfb..f5cce5c 100644
--- a/pkgs/markdown/test/gfm/backslash_escapes.unit
+++ b/pkgs/markdown/test/gfm/backslash_escapes.unit
@@ -75,5 +75,5 @@
foo
```
<<<
-<pre><code class="language-foo\+bar">foo
+<pre><code class="language-foo+bar">foo
</code></pre>
diff --git a/pkgs/markdown/test/gfm/block_quotes.unit b/pkgs/markdown/test/gfm/block_quotes.unit
index 5c45af4..30e1ef9 100644
--- a/pkgs/markdown/test/gfm/block_quotes.unit
+++ b/pkgs/markdown/test/gfm/block_quotes.unit
@@ -96,8 +96,7 @@
<pre><code>foo
</code></pre>
</blockquote>
-<pre><code>
-</code></pre>
+<pre><code></code></pre>
>>> Block quotes - 216
> foo
- bar
diff --git a/pkgs/markdown/test/gfm/fenced_code_blocks.unit b/pkgs/markdown/test/gfm/fenced_code_blocks.unit
index 11ab448..dd13ca0 100644
--- a/pkgs/markdown/test/gfm/fenced_code_blocks.unit
+++ b/pkgs/markdown/test/gfm/fenced_code_blocks.unit
@@ -61,8 +61,7 @@
>>> Fenced code blocks - 96
```
<<<
-<pre><code>
-</code></pre>
+<pre><code></code></pre>
>>> Fenced code blocks - 97
`````
@@ -72,7 +71,6 @@
<pre><code>
```
aaa
-
</code></pre>
>>> Fenced code blocks - 98
> ```
@@ -105,7 +103,7 @@
aaa
```
<<<
-<pre><code> aaa
+<pre><code>aaa
aaa
</code></pre>
>>> Fenced code blocks - 102
@@ -116,7 +114,7 @@
```
<<<
<pre><code>aaa
- aaa
+aaa
aaa
</code></pre>
>>> Fenced code blocks - 103
@@ -126,9 +124,9 @@
aaa
```
<<<
-<pre><code> aaa
- aaa
- aaa
+<pre><code>aaa
+ aaa
+aaa
</code></pre>
>>> Fenced code blocks - 104
```
@@ -160,7 +158,6 @@
<<<
<pre><code>aaa
```
-
</code></pre>
>>> Fenced code blocks - 108
``` ```
@@ -175,7 +172,6 @@
<<<
<pre><code>aaa
~~~ ~~
-
</code></pre>
>>> Fenced code blocks - 110
foo
@@ -245,6 +241,5 @@
``` aaa
```
<<<
-<pre><code></code></pre>
-<pre><code>
+<pre><code>``` aaa
</code></pre>
diff --git a/pkgs/markdown/test/gfm/lists.unit b/pkgs/markdown/test/gfm/lists.unit
index 6a6e847..9911dea 100644
--- a/pkgs/markdown/test/gfm/lists.unit
+++ b/pkgs/markdown/test/gfm/lists.unit
@@ -279,7 +279,6 @@
</li>
</ul>
<pre><code>- c
-
</code></pre>
>>> Lists - 299
- a
diff --git a/pkgs/markdown/tool/common_mark_stats.json b/pkgs/markdown/tool/common_mark_stats.json
index 7c98634..47935d8 100644
--- a/pkgs/markdown/tool/common_mark_stats.json
+++ b/pkgs/markdown/tool/common_mark_stats.json
@@ -53,7 +53,7 @@
"21": "strict",
"22": "strict",
"23": "strict",
- "24": "fail"
+ "24": "strict"
},
"Blank lines": {
"227": "strict"
@@ -269,20 +269,20 @@
"123": "strict",
"124": "strict",
"125": "strict",
- "126": "loose",
- "127": "loose",
+ "126": "strict",
+ "127": "strict",
"128": "strict",
"129": "strict",
"130": "strict",
- "131": "loose",
- "132": "loose",
- "133": "loose",
+ "131": "strict",
+ "132": "strict",
+ "133": "strict",
"134": "strict",
"135": "strict",
"136": "strict",
- "137": "loose",
+ "137": "strict",
"138": "loose",
- "139": "loose",
+ "139": "strict",
"140": "strict",
"141": "strict",
"142": "strict",
@@ -290,7 +290,7 @@
"144": "strict",
"145": "strict",
"146": "strict",
- "147": "fail"
+ "147": "strict"
},
"Hard line breaks": {
"633": "strict",
diff --git a/pkgs/markdown/tool/common_mark_stats.txt b/pkgs/markdown/tool/common_mark_stats.txt
index f1200e9..27d1105 100644
--- a/pkgs/markdown/tool/common_mark_stats.txt
+++ b/pkgs/markdown/tool/common_mark_stats.txt
@@ -1,12 +1,12 @@
17 of 18 – 94.4% ATX headings
19 of 19 – 100.0% Autolinks
- 12 of 13 – 92.3% Backslash escapes
+ 13 of 13 – 100.0% Backslash escapes
1 of 1 – 100.0% Blank lines
23 of 25 – 92.0% Block quotes
22 of 22 – 100.0% Code spans
130 of 131 – 99.2% Emphasis and strong emphasis
15 of 17 – 88.2% Entity and numeric character references
- 28 of 29 – 96.6% Fenced code blocks
+ 29 of 29 – 100.0% Fenced code blocks
15 of 15 – 100.0% Hard line breaks
44 of 44 – 100.0% HTML blocks
21 of 22 – 95.5% Images
@@ -24,5 +24,5 @@
11 of 11 – 100.0% Tabs
3 of 3 – 100.0% Textual content
19 of 19 – 100.0% Thematic breaks
- 622 of 652 – 95.4% TOTAL
- 562 of 622 – 90.4% TOTAL Strict
+ 624 of 652 – 95.7% TOTAL
+ 571 of 624 – 91.5% TOTAL Strict
diff --git a/pkgs/markdown/tool/gfm_stats.json b/pkgs/markdown/tool/gfm_stats.json
index c37aada..01e91ab 100644
--- a/pkgs/markdown/tool/gfm_stats.json
+++ b/pkgs/markdown/tool/gfm_stats.json
@@ -66,7 +66,7 @@
"317": "strict",
"318": "strict",
"319": "strict",
- "320": "fail"
+ "320": "strict"
},
"Blank lines": {
"197": "strict"
@@ -285,20 +285,20 @@
"93": "strict",
"94": "strict",
"95": "strict",
- "96": "loose",
- "97": "loose",
+ "96": "strict",
+ "97": "strict",
"98": "strict",
"99": "strict",
"100": "strict",
- "101": "loose",
- "102": "loose",
- "103": "loose",
+ "101": "strict",
+ "102": "strict",
+ "103": "strict",
"104": "strict",
"105": "strict",
"106": "strict",
- "107": "loose",
+ "107": "strict",
"108": "loose",
- "109": "loose",
+ "109": "strict",
"110": "strict",
"111": "strict",
"112": "strict",
@@ -306,7 +306,7 @@
"114": "strict",
"115": "strict",
"116": "strict",
- "117": "fail"
+ "117": "strict"
},
"Hard line breaks": {
"654": "strict",
diff --git a/pkgs/markdown/tool/gfm_stats.txt b/pkgs/markdown/tool/gfm_stats.txt
index 5b47d05..eff6bbc 100644
--- a/pkgs/markdown/tool/gfm_stats.txt
+++ b/pkgs/markdown/tool/gfm_stats.txt
@@ -1,14 +1,14 @@
17 of 18 – 94.4% ATX headings
19 of 19 – 100.0% Autolinks
11 of 11 – 100.0% Autolinks (extension)
- 12 of 13 – 92.3% Backslash escapes
+ 13 of 13 – 100.0% Backslash escapes
1 of 1 – 100.0% Blank lines
23 of 25 – 92.0% Block quotes
22 of 22 – 100.0% Code spans
0 of 1 – 0.0% Disallowed Raw HTML (extension)
130 of 131 – 99.2% Emphasis and strong emphasis
15 of 17 – 88.2% Entity and numeric character references
- 28 of 29 – 96.6% Fenced code blocks
+ 29 of 29 – 100.0% Fenced code blocks
15 of 15 – 100.0% Hard line breaks
43 of 43 – 100.0% HTML blocks
21 of 22 – 95.5% Images
@@ -28,5 +28,5 @@
11 of 11 – 100.0% Tabs
3 of 3 – 100.0% Textual content
19 of 19 – 100.0% Thematic breaks
- 640 of 671 – 95.4% TOTAL
- 578 of 640 – 90.3% TOTAL Strict
+ 642 of 671 – 95.7% TOTAL
+ 587 of 642 – 91.4% TOTAL Strict
