Overhaul the parsing of emphasis and strong emphasis (#193)
Overhaul the parsing of emphasis and strong emphasis
CommonMark's emphasis and strong emphasis require complex definitions of
"delimiter runs" as well as 17 parsing rules. This implementation was emulating
these rules poorly with some simple regular expressions.
This work examines possible matches for opening and closing (strong) emphasis in
terms of "delimiter runs" and ability to "open emphasis" and "close emphasis",
in order to dramatically improve the spec-matching for Emphasis.
No specs flip from strict/loose to fail with this work.
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b1c50d5..96f5b64 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 1.1.2-dev
+
+* Overhaul support for emphasis (`*foo*` and `_foo_`) and strong emphasis
+ (`**foo**` and `__foo__`). This raises the compliance with the CommonMark
+ specs to 89%, and the compliance with the GFM specs to 86%.
+
## 1.1.1
* Add support for GitHub's colon-based Emoji syntax. :tada:! This is available
diff --git a/lib/src/inline_parser.dart b/lib/src/inline_parser.dart
index a5e13a8..fb6e8df 100644
--- a/lib/src/inline_parser.dart
+++ b/lib/src/inline_parser.dart
@@ -30,14 +30,10 @@
new TextSyntax(r'&', sub: '&'),
// Encode "<". (Why not encode ">" too? Gruber is toying with us.)
new TextSyntax(r'<', sub: '<'),
- // Parse "**strong**" tags.
- new TagSyntax(r'\*\*', tag: 'strong'),
- // Parse "__strong__" tags.
- new TagSyntax(r'\b__', tag: 'strong', end: r'__\b'),
- // Parse "*emphasis*" tags.
- new TagSyntax(r'\*', tag: 'em'),
- // Parse "_emphasis_" tags.
- new TagSyntax(r'\b_', tag: 'em', end: r'_\b'),
+ // Parse "**strong**" and "*emphasis*" tags.
+ new TagSyntax(r'\*+', requiresDelimiterRun: true),
+ // Parse "__strong__" and "_emphasis_" tags.
+ new TagSyntax(r'_+', requiresDelimiterRun: true),
new CodeSyntax(),
// We will add the LinkSyntax once we know about the specific link resolver.
]);
@@ -88,7 +84,7 @@
List<Node> parse() {
// Make a fake top tag to hold the results.
- _stack.add(new TagState(0, 0, null));
+ _stack.add(new TagState(0, 0, null, null));
while (!isDone) {
var matched = false;
@@ -278,24 +274,158 @@
}
}
+class _DelimiterRun {
+ static final String punctuation = r'''!"#$%&'()*+,-./:;<=>?@[\]^_`{|}~''';
+ // TODO(srawlins): Unicode whitespace
+ static final String whitespace = ' \t\r\n';
+
+ final String char;
+ final int length;
+ final bool isLeftFlanking;
+ final bool isRightFlanking;
+ final bool isPrecededByPunctuation;
+ final bool isFollowedByPunctuation;
+
+ _DelimiterRun._(
+ {this.char,
+ this.length,
+ this.isLeftFlanking,
+ this.isRightFlanking,
+ this.isPrecededByPunctuation,
+ this.isFollowedByPunctuation});
+
+ static _DelimiterRun tryParse(InlineParser parser, int runStart, int runEnd) {
+ bool leftFlanking,
+ rightFlanking,
+ precededByPunctuation,
+ followedByPunctuation;
+ String preceding, following;
+ if (runStart == 0) {
+ rightFlanking = false;
+ preceding = '\n';
+ } else {
+ preceding = parser.source.substring(runStart - 1, runStart);
+ }
+ precededByPunctuation = punctuation.contains(preceding);
+
+ if (runEnd == parser.source.length - 1) {
+ leftFlanking = false;
+ following = '\n';
+ } else {
+ following = parser.source.substring(runEnd + 1, runEnd + 2);
+ }
+ followedByPunctuation = punctuation.contains(following);
+
+ // http://spec.commonmark.org/0.28/#left-flanking-delimiter-run
+ if (whitespace.contains(following)) {
+ leftFlanking = false;
+ } else {
+ leftFlanking = !followedByPunctuation ||
+ whitespace.contains(preceding) ||
+ precededByPunctuation;
+ }
+
+ // http://spec.commonmark.org/0.28/#right-flanking-delimiter-run
+ if (whitespace.contains(preceding)) {
+ rightFlanking = false;
+ } else {
+ rightFlanking = !precededByPunctuation ||
+ whitespace.contains(following) ||
+ followedByPunctuation;
+ }
+
+ if (!leftFlanking && !rightFlanking) {
+ // Could not parse a delimiter run.
+ return null;
+ }
+
+ return new _DelimiterRun._(
+ char: parser.source.substring(runStart, runStart + 1),
+ length: runEnd - runStart + 1,
+ isLeftFlanking: leftFlanking,
+ isRightFlanking: rightFlanking,
+ isPrecededByPunctuation: precededByPunctuation,
+ isFollowedByPunctuation: followedByPunctuation);
+ }
+
+ String toString() =>
+ '<char: $char, length: $length, isLeftFlanking: $isLeftFlanking, '
+ 'isRightFlanking: $isRightFlanking>';
+
+ // Whether a delimiter in this run can open emphasis or strong emphasis.
+ bool get canOpen =>
+ isLeftFlanking &&
+ (char == '*' || !isRightFlanking || isPrecededByPunctuation);
+
+ // Whether a delimiter in this run can close emphasis or strong emphasis.
+ bool get canClose =>
+ isRightFlanking &&
+ (char == '*' || !isLeftFlanking || isFollowedByPunctuation);
+}
+
/// Matches syntax that has a pair of tags and becomes an element, like `*` for
/// `<em>`. Allows nested tags.
class TagSyntax extends InlineSyntax {
final RegExp endPattern;
- final String tag;
+ final bool requiresDelimiterRun;
- TagSyntax(String pattern, {this.tag, String end})
+ TagSyntax(String pattern, {String end, this.requiresDelimiterRun: false})
: endPattern = new RegExp((end != null) ? end : pattern, multiLine: true),
super(pattern);
bool onMatch(InlineParser parser, Match match) {
- parser._stack
- .add(new TagState(parser.pos, parser.pos + match[0].length, this));
- return true;
+ var runLength = match.group(0).length;
+ var matchStart = parser.pos;
+ var matchEnd = parser.pos + runLength - 1;
+ var delimiterRun = _DelimiterRun.tryParse(parser, matchStart, matchEnd);
+ if (delimiterRun != null && delimiterRun.canOpen) {
+ parser._stack
+ .add(new TagState(parser.pos, matchEnd + 1, this, delimiterRun));
+ return true;
+ } else {
+ parser.advanceBy(runLength);
+ return false;
+ }
}
bool onMatchEnd(InlineParser parser, Match match, TagState state) {
- parser.addNode(new Element(tag, state.children));
+ var runLength = match.group(0).length;
+ var matchStart = parser.pos;
+ var matchEnd = parser.pos + runLength - 1;
+ var openingRunLength = state.endPos - state.startPos;
+ var delimiterRun = _DelimiterRun.tryParse(parser, matchStart, matchEnd);
+ if (!delimiterRun.isRightFlanking) {
+ return false;
+ }
+
+ if (openingRunLength == 1 && runLength == 1) {
+ parser.addNode(new Element('em', state.children));
+ } else if (openingRunLength == 1 && runLength > 1) {
+ parser.addNode(new Element('em', state.children));
+ parser.pos = parser.pos - (runLength - 1);
+ parser.start = parser.pos;
+ } else if (openingRunLength > 1 && runLength == 1) {
+ parser._stack.add(
+ new TagState(state.startPos, state.endPos - 1, this, delimiterRun));
+ parser.addNode(new Element('em', state.children));
+ } else if (openingRunLength == 2 && runLength == 2) {
+ parser.addNode(new Element('strong', state.children));
+ } else if (openingRunLength == 2 && runLength > 2) {
+ parser.addNode(new Element('strong', state.children));
+ parser.pos = parser.pos - (runLength - 2);
+ parser.start = parser.pos;
+ } else if (openingRunLength > 2 && runLength == 2) {
+ parser._stack.add(
+ new TagState(state.startPos, state.endPos - 2, this, delimiterRun));
+ parser.addNode(new Element('strong', state.children));
+ } else if (openingRunLength > 2 && runLength > 2) {
+ parser._stack.add(
+ new TagState(state.startPos, state.endPos - 2, this, delimiterRun));
+ parser.addNode(new Element('strong', state.children));
+ parser.pos = parser.pos - (runLength - 2);
+ parser.start = parser.pos;
+ }
+
return true;
}
}
@@ -519,19 +649,46 @@
/// The children of this node. Will be `null` for text nodes.
final List<Node> children;
- TagState(this.startPos, this.endPos, this.syntax) : children = <Node>[];
+ final _DelimiterRun openingDelimiterRun;
+
+ TagState(this.startPos, this.endPos, this.syntax, this.openingDelimiterRun)
+ : children = <Node>[];
/// Attempts to close this tag by matching the current text against its end
/// pattern.
bool tryMatch(InlineParser parser) {
var endMatch = syntax.endPattern.matchAsPrefix(parser.source, parser.pos);
- if (endMatch != null) {
+ if (endMatch == null) {
+ return false;
+ }
+
+ if (!syntax.requiresDelimiterRun) {
// Close the tag.
close(parser, endMatch);
return true;
}
- return false;
+ var runLength = endMatch.group(0).length;
+ var openingRunLength = endPos - startPos;
+ var closingMatchStart = parser.pos;
+ var closingMatchEnd = parser.pos + runLength - 1;
+ var closingDelimiterRun =
+ _DelimiterRun.tryParse(parser, closingMatchStart, closingMatchEnd);
+ if (closingDelimiterRun != null && closingDelimiterRun.canClose) {
+ // Emphasis rules #9 and #10:
+ var oneRunOpensAndCloses =
+ (openingDelimiterRun.canOpen && openingDelimiterRun.canClose) ||
+ (closingDelimiterRun.canOpen && closingDelimiterRun.canClose);
+ if (oneRunOpensAndCloses &&
+ (openingRunLength + closingDelimiterRun.length) % 3 == 0) {
+ return false;
+ }
+ // Close the tag.
+ close(parser, endMatch);
+ return true;
+ } else {
+ return false;
+ }
}
/// Pops this tag off the stack, completes it, and adds it to the output.
diff --git a/lib/src/version.dart b/lib/src/version.dart
index 5f432d2..a4966e3 100644
--- a/lib/src/version.dart
+++ b/lib/src/version.dart
@@ -1,2 +1,2 @@
/// The current version of markdown.
-final String version = '1.1.1';
+final String version = '1.1.2-dev';
diff --git a/pubspec.yaml b/pubspec.yaml
index 7f69f6e..d7e97b8 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -1,5 +1,5 @@
name: markdown
-version: 1.1.1
+version: 1.1.2-dev
author: Dart Team <misc@dartlang.org>
description: A library for converting markdown to HTML.
homepage: https://github.com/dart-lang/markdown
diff --git a/test/original/emphasis_and_strong.unit b/test/original/emphasis_and_strong.unit
index 90aea56..13a6dd2 100644
--- a/test/original/emphasis_and_strong.unit
+++ b/test/original/emphasis_and_strong.unit
@@ -65,11 +65,6 @@
<<<
<p><em>a _b</em> c_</p>
->>> cannot nest tags of same type
-*a _b *c* d_ e*
-
-<<<
-<p><em>a _b </em>c<em> d_ e</em></p>
>>> in the middle of a word
a_b_c a__b__c a*b*c a**b**c
<<<
@@ -85,4 +80,4 @@
>>> spanning words
_a_b c_d_ __a__b c__d__ *a*b c*d* **a**b c**d**
<<<
-<p><em>a_b c_d</em> <strong>a__b c__d</strong> <em>a</em>b c<em>d</em> <strong>a</strong>b c<strong>d</strong></p>
\ No newline at end of file
+<p><em>a_b c_d</em> <strong>a__b c__d</strong> <em>a</em>b c<em>d</em> <strong>a</strong>b c<strong>d</strong></p>
diff --git a/tool/common_mark_stats.json b/tool/common_mark_stats.json
index ec0674c..83b6f50 100644
--- a/tool/common_mark_stats.json
+++ b/tool/common_mark_stats.json
@@ -107,123 +107,123 @@
"Emphasis and strong emphasis": {
"331": "strict",
"332": "strict",
- "333": "fail",
+ "333": "loose",
"334": "fail",
"335": "strict",
"336": "strict",
"337": "strict",
- "338": "fail",
+ "338": "strict",
"339": "loose",
"340": "strict",
"341": "strict",
- "342": "fail",
+ "342": "strict",
"343": "loose",
"344": "strict",
"345": "strict",
- "346": "fail",
+ "346": "strict",
"347": "fail",
- "348": "fail",
- "349": "fail",
+ "348": "strict",
+ "349": "strict",
"350": "strict",
- "351": "fail",
+ "351": "strict",
"352": "strict",
"353": "strict",
"354": "strict",
- "355": "fail",
+ "355": "strict",
"356": "strict",
"357": "strict",
"358": "strict",
- "359": "fail",
- "360": "fail",
+ "359": "strict",
+ "360": "loose",
"361": "strict",
"362": "strict",
- "363": "fail",
- "364": "fail",
+ "363": "strict",
+ "364": "strict",
"365": "loose",
"366": "strict",
"367": "strict",
- "368": "fail",
+ "368": "strict",
"369": "strict",
"370": "strict",
- "371": "fail",
- "372": "fail",
- "373": "fail",
+ "371": "strict",
+ "372": "strict",
+ "373": "strict",
"374": "strict",
"375": "loose",
"376": "strict",
- "377": "fail",
+ "377": "strict",
"378": "strict",
"379": "strict",
"380": "strict",
- "381": "fail",
+ "381": "strict",
"382": "strict",
"383": "strict",
"384": "strict",
"385": "strict",
"386": "strict",
"387": "strict",
- "388": "fail",
- "389": "fail",
- "390": "fail",
- "391": "fail",
- "392": "fail",
- "393": "fail",
- "394": "fail",
- "395": "fail",
- "396": "fail",
+ "388": "strict",
+ "389": "strict",
+ "390": "strict",
+ "391": "strict",
+ "392": "strict",
+ "393": "strict",
+ "394": "strict",
+ "395": "strict",
+ "396": "strict",
"397": "strict",
- "398": "fail",
+ "398": "strict",
"399": "strict",
"400": "strict",
"401": "strict",
"402": "strict",
- "403": "fail",
- "404": "fail",
+ "403": "strict",
+ "404": "strict",
"405": "strict",
"406": "strict",
"407": "strict",
"408": "strict",
- "409": "fail",
+ "409": "strict",
"410": "strict",
"411": "strict",
- "412": "fail",
+ "412": "strict",
"413": "strict",
"414": "strict",
"415": "strict",
- "416": "fail",
+ "416": "strict",
"417": "strict",
"418": "strict",
- "419": "fail",
+ "419": "strict",
"420": "strict",
- "421": "fail",
- "422": "fail",
+ "421": "strict",
+ "422": "strict",
"423": "strict",
"424": "strict",
"425": "strict",
"426": "strict",
"427": "strict",
- "428": "fail",
+ "428": "strict",
"429": "strict",
"430": "strict",
- "431": "fail",
- "432": "loose",
- "433": "loose",
- "434": "fail",
- "435": "loose",
- "436": "loose",
+ "431": "strict",
+ "432": "strict",
+ "433": "strict",
+ "434": "strict",
+ "435": "strict",
+ "436": "strict",
"437": "strict",
"438": "strict",
"439": "strict",
"440": "strict",
- "441": "fail",
- "442": "fail",
- "443": "fail",
- "444": "fail",
- "445": "fail",
+ "441": "strict",
+ "442": "strict",
+ "443": "strict",
+ "444": "strict",
+ "445": "strict",
"446": "strict",
- "447": "fail",
- "448": "loose",
- "449": "loose",
+ "447": "strict",
+ "448": "strict",
+ "449": "strict",
"450": "fail",
"451": "fail",
"452": "strict",
@@ -435,7 +435,7 @@
"484": "strict",
"485": "strict",
"486": "strict",
- "487": "fail",
+ "487": "strict",
"488": "loose",
"489": "fail",
"490": "fail",
@@ -449,7 +449,7 @@
"498": "strict",
"499": "loose",
"500": "strict",
- "501": "fail",
+ "501": "strict",
"502": "loose",
"503": "loose",
"504": "fail",
@@ -667,7 +667,7 @@
"22": "strict",
"23": "strict",
"24": "strict",
- "25": "fail",
+ "25": "strict",
"26": "loose",
"27": "loose",
"28": "strict",
diff --git a/tool/common_mark_stats.txt b/tool/common_mark_stats.txt
index 03e10f4..f6edbe5 100644
--- a/tool/common_mark_stats.txt
+++ b/tool/common_mark_stats.txt
@@ -4,7 +4,7 @@
1 of 1 – 100.0% Blank lines
22 of 25 – 88.0% Block quotes
16 of 17 – 94.1% Code spans
- 79 of 128 – 61.7% Emphasis and strong emphasis
+ 124 of 128 – 96.9% Emphasis and strong emphasis
10 of 12 – 83.3% Entity and numeric character references
25 of 28 – 89.3% Fenced code blocks
15 of 15 – 100.0% Hard line breaks
@@ -13,7 +13,7 @@
11 of 12 – 91.7% Indented code blocks
1 of 1 – 100.0% Inlines
21 of 23 – 91.3% Link reference definitions
- 58 of 84 – 69.0% Links
+ 60 of 84 – 71.4% Links
44 of 48 – 91.7% List items
18 of 24 – 75.0% Lists
8 of 8 – 100.0% Paragraphs
@@ -23,5 +23,5 @@
2 of 2 – 100.0% Soft line breaks
6 of 11 – 54.5% Tabs
3 of 3 – 100.0% Textual content
- 17 of 19 – 89.5% Thematic breaks
- 507 of 624 – 81.3% TOTAL
+ 18 of 19 – 94.7% Thematic breaks
+ 555 of 624 – 88.9% TOTAL
diff --git a/tool/gfm_stats.json b/tool/gfm_stats.json
index 451db60..5d32124 100644
--- a/tool/gfm_stats.json
+++ b/tool/gfm_stats.json
@@ -123,123 +123,123 @@
"Emphasis and strong emphasis": {
"340": "strict",
"341": "strict",
- "342": "fail",
+ "342": "loose",
"343": "fail",
"344": "strict",
"345": "strict",
"346": "strict",
- "347": "fail",
+ "347": "strict",
"348": "loose",
"349": "strict",
"350": "strict",
- "351": "fail",
+ "351": "strict",
"352": "loose",
"353": "strict",
"354": "strict",
- "355": "fail",
+ "355": "strict",
"356": "fail",
- "357": "fail",
- "358": "fail",
+ "357": "strict",
+ "358": "strict",
"359": "strict",
- "360": "fail",
+ "360": "strict",
"361": "strict",
"362": "strict",
"363": "strict",
- "364": "fail",
+ "364": "strict",
"365": "strict",
"366": "strict",
"367": "strict",
- "368": "fail",
- "369": "fail",
+ "368": "strict",
+ "369": "loose",
"370": "strict",
"371": "strict",
- "372": "fail",
- "373": "fail",
+ "372": "strict",
+ "373": "strict",
"374": "loose",
"375": "strict",
"376": "strict",
- "377": "fail",
+ "377": "strict",
"378": "strict",
"379": "strict",
- "380": "fail",
- "381": "fail",
- "382": "fail",
+ "380": "strict",
+ "381": "strict",
+ "382": "strict",
"383": "strict",
"384": "loose",
"385": "strict",
- "386": "fail",
+ "386": "strict",
"387": "strict",
"388": "strict",
"389": "strict",
- "390": "fail",
+ "390": "strict",
"391": "strict",
"392": "strict",
"393": "strict",
"394": "strict",
"395": "strict",
"396": "strict",
- "397": "fail",
- "398": "fail",
- "399": "fail",
- "400": "fail",
- "401": "fail",
- "402": "fail",
- "403": "fail",
- "404": "fail",
- "405": "fail",
+ "397": "strict",
+ "398": "strict",
+ "399": "strict",
+ "400": "strict",
+ "401": "strict",
+ "402": "strict",
+ "403": "strict",
+ "404": "strict",
+ "405": "strict",
"406": "strict",
- "407": "fail",
+ "407": "strict",
"408": "strict",
"409": "strict",
"410": "strict",
"411": "strict",
- "412": "fail",
- "413": "fail",
+ "412": "strict",
+ "413": "strict",
"414": "strict",
"415": "strict",
"416": "strict",
"417": "strict",
- "418": "fail",
+ "418": "strict",
"419": "strict",
"420": "strict",
- "421": "fail",
+ "421": "strict",
"422": "strict",
"423": "strict",
"424": "strict",
- "425": "fail",
+ "425": "strict",
"426": "strict",
"427": "strict",
- "428": "fail",
+ "428": "strict",
"429": "strict",
- "430": "fail",
- "431": "fail",
+ "430": "strict",
+ "431": "strict",
"432": "strict",
"433": "strict",
"434": "strict",
"435": "strict",
"436": "strict",
- "437": "fail",
+ "437": "strict",
"438": "strict",
"439": "strict",
- "440": "fail",
- "441": "loose",
- "442": "loose",
- "443": "fail",
- "444": "loose",
- "445": "loose",
+ "440": "strict",
+ "441": "strict",
+ "442": "strict",
+ "443": "strict",
+ "444": "strict",
+ "445": "strict",
"446": "strict",
"447": "strict",
"448": "strict",
"449": "strict",
- "450": "fail",
- "451": "fail",
- "452": "fail",
- "453": "fail",
- "454": "fail",
+ "450": "strict",
+ "451": "strict",
+ "452": "strict",
+ "453": "strict",
+ "454": "strict",
"455": "strict",
- "456": "fail",
- "457": "loose",
- "458": "loose",
+ "456": "strict",
+ "457": "strict",
+ "458": "strict",
"459": "fail",
"460": "fail",
"461": "strict",
@@ -451,7 +451,7 @@
"496": "strict",
"497": "strict",
"498": "strict",
- "499": "fail",
+ "499": "strict",
"500": "loose",
"501": "fail",
"502": "fail",
@@ -465,7 +465,7 @@
"510": "strict",
"511": "loose",
"512": "strict",
- "513": "fail",
+ "513": "strict",
"514": "loose",
"515": "loose",
"516": "fail",
@@ -697,7 +697,7 @@
"22": "strict",
"23": "strict",
"24": "strict",
- "25": "fail",
+ "25": "strict",
"26": "loose",
"27": "loose",
"28": "strict",
diff --git a/tool/gfm_stats.txt b/tool/gfm_stats.txt
index 7bf50f1..08bfcef 100644
--- a/tool/gfm_stats.txt
+++ b/tool/gfm_stats.txt
@@ -6,7 +6,7 @@
22 of 25 – 88.0% Block quotes
16 of 17 – 94.1% Code spans
0 of 1 – 0.0% Disallowed Raw HTML (extension)
- 79 of 128 – 61.7% Emphasis and strong emphasis
+ 124 of 128 – 96.9% Emphasis and strong emphasis
10 of 12 – 83.3% Entity and numeric character references
25 of 28 – 89.3% Fenced code blocks
15 of 15 – 100.0% Hard line breaks
@@ -15,7 +15,7 @@
11 of 12 – 91.7% Indented code blocks
1 of 1 – 100.0% Inlines
21 of 23 – 91.3% Link reference definitions
- 58 of 84 – 69.0% Links
+ 60 of 84 – 71.4% Links
44 of 48 – 91.7% List items
18 of 24 – 75.0% Lists
8 of 8 – 100.0% Paragraphs
@@ -27,5 +27,5 @@
2 of 7 – 28.6% Tables (extension)
6 of 11 – 54.5% Tabs
3 of 3 – 100.0% Textual content
- 17 of 19 – 89.5% Thematic breaks
- 510 of 646 – 78.9% TOTAL
+ 18 of 19 – 94.7% Thematic breaks
+ 558 of 646 – 86.4% TOTAL
