Refactor dartdoc parsing
This extracts and refactors a sizable chunk of dartdoc parsing out of AstBuilder
and into the fasta parser. Parsing dartdoc does not happen unless the listener
calls the new fasta parser parseCommentReferences method.
The new parseCommentReferences method generates a handleCommentReferenceText
event for each comment reference encountered. The listener calls the scanner
to tokenize the comment reference and forwards the result to a new
fasta parser parseOneCommentReference method. This method generates either
a handleCommentReference or a handleNoCommentReference depending upon
whether or not a reference is parsed.
parser.parseCommentReferences
* generates handleCommentReferenceText events
* returns # of events generated
listener.handleCommentReferenceText
* calls parser.parseOneCommentReference
parser.parseOneCommentReference
* generates either handleCommentReference or handleNoCommentReference
There are further improvements to be made, but I believe this chunk
will allow progress to be made on the kernel side.
Change-Id: I393dc4d1d4791b3d7a529f6cb3c16db3a5befddf
Reviewed-on: https://dart-review.googlesource.com/68461
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
diff --git a/pkg/analyzer/lib/src/fasta/ast_builder.dart b/pkg/analyzer/lib/src/fasta/ast_builder.dart
index 55dda56..969e984 100644
--- a/pkg/analyzer/lib/src/fasta/ast_builder.dart
+++ b/pkg/analyzer/lib/src/fasta/ast_builder.dart
@@ -8,8 +8,6 @@
import 'package:analyzer/dart/ast/token.dart' show Token, TokenType;
import 'package:analyzer/error/listener.dart';
import 'package:analyzer/src/fasta/error_converter.dart';
-import 'package:analyzer/src/generated/java_core.dart';
-import 'package:analyzer/src/generated/java_engine.dart';
import 'package:analyzer/src/generated/utilities_dart.dart';
import 'package:front_end/src/fasta/parser.dart'
show
@@ -22,12 +20,7 @@
import 'package:front_end/src/fasta/scanner.dart' hide StringToken;
import 'package:front_end/src/scanner/errors.dart' show translateErrorToken;
import 'package:front_end/src/scanner/token.dart'
- show
- BeginToken,
- StringToken,
- SyntheticBeginToken,
- SyntheticStringToken,
- SyntheticToken;
+ show BeginToken, SyntheticBeginToken, SyntheticStringToken, SyntheticToken;
import 'package:front_end/src/fasta/problems.dart' show unhandled;
import 'package:front_end/src/fasta/messages.dart'
@@ -2566,6 +2559,38 @@
push(popTypedList<Annotation>(count) ?? NullValue.Metadata);
}
+ @override
+ void handleCommentReferenceText(
+ Token commentToken, String referenceSource, int referenceOffset) {
+ ScannerResult result = scanString(referenceSource);
+ if (result.hasErrors) {
+ handleNoCommentReference();
+ } else {
+ Token token = result.tokens;
+ do {
+ token.offset += referenceOffset;
+ token = token.next;
+ } while (!token.isEof);
+ parser.parseOneCommentReference(commentToken, result.tokens);
+ }
+ }
+
+ @override
+ void handleCommentReference(
+ Token newKeyword, Token prefix, Token period, Token token) {
+ Identifier identifier = ast.simpleIdentifier(token);
+ if (prefix != null) {
+ identifier = ast.prefixedIdentifier(
+ ast.simpleIdentifier(prefix), period, identifier);
+ }
+ push(ast.commentReference(newKeyword, identifier));
+ }
+
+ @override
+ void handleNoCommentReference() {
+ push(NullValue.CommentReference);
+ }
+
ParameterKind _toAnalyzerParameterKind(FormalParameterKind type) {
if (type == FormalParameterKind.optionalPositional) {
return ParameterKind.POSITIONAL;
@@ -2600,306 +2625,25 @@
return null;
}
- /// Search the given list of [ranges] for a range that contains the given
- /// [index]. Return the range that was found, or `null` if none of the ranges
- /// contain the index.
- List<int> _findRange(List<List<int>> ranges, int index) {
- int rangeCount = ranges.length;
- for (int i = 0; i < rangeCount; i++) {
- List<int> range = ranges[i];
- if (range[0] <= index && index <= range[1]) {
- return range;
- } else if (index < range[0]) {
- return null;
- }
- }
- return null;
- }
-
- /// Return a list of the ranges of characters in the given [comment] that
- /// should be treated as code blocks.
- List<List<int>> _getCodeBlockRanges(String comment) {
- List<List<int>> ranges = <List<int>>[];
- int length = comment.length;
- if (length < 3) {
- return ranges;
- }
+ /// Remove any substrings in the given [comment] that represent in-line code
+ /// in markdown.
+ String removeInlineCodeBlocks(String comment) {
int index = 0;
- int firstChar = comment.codeUnitAt(0);
- if (firstChar == 0x2F) {
- int secondChar = comment.codeUnitAt(1);
- int thirdChar = comment.codeUnitAt(2);
- if ((secondChar == 0x2A && thirdChar == 0x2A) ||
- (secondChar == 0x2F && thirdChar == 0x2F)) {
- index = 3;
+ while (true) {
+ int beginIndex = comment.indexOf('`', index);
+ if (beginIndex == -1) {
+ break;
}
+ int endIndex = comment.indexOf('`', beginIndex + 1);
+ if (endIndex == -1) {
+ break;
+ }
+ comment = comment.substring(0, beginIndex + 1) +
+ ' ' * (endIndex - beginIndex - 1) +
+ comment.substring(endIndex);
+ index = endIndex + 1;
}
- if (comment.startsWith(' ', index)) {
- int end = index + 4;
- while (end < length &&
- comment.codeUnitAt(end) != 0xD &&
- comment.codeUnitAt(end) != 0xA) {
- end = end + 1;
- }
- ranges.add(<int>[index, end]);
- index = end;
- }
- while (index < length) {
- int currentChar = comment.codeUnitAt(index);
- if (currentChar == 0xD || currentChar == 0xA) {
- index = index + 1;
- while (index < length &&
- Character.isWhitespace(comment.codeUnitAt(index))) {
- index = index + 1;
- }
- if (comment.startsWith(' ', index)) {
- int end = index + 6;
- while (end < length &&
- comment.codeUnitAt(end) != 0xD &&
- comment.codeUnitAt(end) != 0xA) {
- end = end + 1;
- }
- ranges.add(<int>[index, end]);
- index = end;
- }
- } else if (index + 1 < length &&
- currentChar == 0x5B &&
- comment.codeUnitAt(index + 1) == 0x3A) {
- int end = comment.indexOf(':]', index + 2);
- if (end < 0) {
- end = length;
- }
- ranges.add(<int>[index, end]);
- index = end + 1;
- } else {
- index = index + 1;
- }
- }
- return ranges;
- }
-
- ///
- /// Given that we have just found bracketed text within the given [comment],
- /// look to see whether that text is (a) followed by a parenthesized link
- /// address, (b) followed by a colon, or (c) followed by optional whitespace
- /// and another square bracket. The [rightIndex] is the index of the right
- /// bracket. Return `true` if the bracketed text is followed by a link
- /// address.
- ///
- /// This method uses the syntax described by the
- /// <a href="http://daringfireball.net/projects/markdown/syntax">markdown</a>
- /// project.
- bool _isLinkText(String comment, int rightIndex) {
- int length = comment.length;
- int index = rightIndex + 1;
- if (index >= length) {
- return false;
- }
- int nextChar = comment.codeUnitAt(index);
- if (nextChar == 0x28 || nextChar == 0x3A) {
- return true;
- }
- while (Character.isWhitespace(nextChar)) {
- index = index + 1;
- if (index >= length) {
- return false;
- }
- nextChar = comment.codeUnitAt(index);
- }
- return nextChar == 0x5B;
- }
-
- /// Parse a comment reference from the source between square brackets. The
- /// [referenceSource] is the source occurring between the square brackets
- /// within a documentation comment. The [sourceOffset] is the offset of the
- /// first character of the reference source. Return the comment reference that
- /// was parsed, or `null` if no reference could be found.
- /// ```
- /// commentReference ::=
- /// 'new'? prefixedIdentifier
- /// ```
- CommentReference _parseCommentReference(
- String referenceSource, int sourceOffset) {
- // TODO(brianwilkerson) The errors are not getting the right offset/length
- // and are being duplicated.
- void offsetTokens(Token token) {
- while (token.type != TokenType.EOF) {
- token.offset = token.offset + sourceOffset;
- token = token.next;
- }
- }
-
- try {
- BooleanErrorListener listener = new BooleanErrorListener();
- ScannerResult result = scanString(referenceSource);
- Token firstToken = result.tokens;
- offsetTokens(firstToken);
- if (listener.errorReported) {
- return null;
- }
- if (firstToken.type == TokenType.EOF) {
- Token syntheticToken =
- new SyntheticStringToken(TokenType.IDENTIFIER, "", sourceOffset);
- syntheticToken.setNext(firstToken);
- return ast.commentReference(null, ast.simpleIdentifier(syntheticToken));
- }
- Token newKeyword = null;
- if (_tokenMatchesKeyword(firstToken, Keyword.NEW)) {
- newKeyword = firstToken;
- firstToken = firstToken.next;
- }
- if (firstToken.isUserDefinableOperator) {
- if (firstToken.next.type != TokenType.EOF) {
- return null;
- }
- Identifier identifier = ast.simpleIdentifier(firstToken);
- return ast.commentReference(null, identifier);
- } else if (_tokenMatchesKeyword(firstToken, Keyword.OPERATOR)) {
- Token secondToken = firstToken.next;
- if (secondToken.isUserDefinableOperator) {
- if (secondToken.next.type != TokenType.EOF) {
- return null;
- }
- Identifier identifier = ast.simpleIdentifier(secondToken);
- return ast.commentReference(null, identifier);
- }
- return null;
- } else if (_tokenMatchesIdentifier(firstToken)) {
- Token secondToken = firstToken.next;
- Token thirdToken = secondToken.next;
- Token nextToken;
- Identifier identifier;
- if (_tokenMatches(secondToken, TokenType.PERIOD)) {
- if (thirdToken.isUserDefinableOperator) {
- identifier = ast.prefixedIdentifier(
- ast.simpleIdentifier(firstToken),
- secondToken,
- ast.simpleIdentifier(thirdToken));
- nextToken = thirdToken.next;
- } else if (_tokenMatchesKeyword(thirdToken, Keyword.OPERATOR)) {
- Token fourthToken = thirdToken.next;
- if (fourthToken.isUserDefinableOperator) {
- identifier = ast.prefixedIdentifier(
- ast.simpleIdentifier(firstToken),
- secondToken,
- ast.simpleIdentifier(fourthToken));
- nextToken = fourthToken.next;
- } else {
- return null;
- }
- } else if (_tokenMatchesIdentifier(thirdToken)) {
- identifier = ast.prefixedIdentifier(
- ast.simpleIdentifier(firstToken),
- secondToken,
- ast.simpleIdentifier(thirdToken));
- nextToken = thirdToken.next;
- }
- } else {
- identifier = ast.simpleIdentifier(firstToken);
- nextToken = firstToken.next;
- }
- if (nextToken.type != TokenType.EOF) {
- return null;
- }
- return ast.commentReference(newKeyword, identifier);
- } else {
- Keyword keyword = firstToken.keyword;
- if (keyword == Keyword.THIS ||
- keyword == Keyword.NULL ||
- keyword == Keyword.TRUE ||
- keyword == Keyword.FALSE) {
- // TODO(brianwilkerson) If we want to support this we will need to
- // extend the definition of CommentReference to take an expression
- // rather than an identifier. For now we just ignore it to reduce the
- // number of errors produced, but that's probably not a valid long
- // term approach.
- return null;
- }
- }
- } catch (exception) {
- // Ignored because we assume that it wasn't a real comment reference.
- }
- return null;
- }
-
- /// Parse all of the comment references occurring in the given array of
- /// documentation comments. The [tokens] are the comment tokens representing
- /// the documentation comments to be parsed. Return the comment references that
- /// were parsed.
- /// ```
- /// commentReference ::=
- /// '[' 'new'? qualified ']' libraryReference?
- ///
- /// libraryReference ::=
- /// '(' stringLiteral ')'
- /// ```
- List<CommentReference> _parseCommentReferences(List<Token> tokens) {
- List<CommentReference> references = <CommentReference>[];
- bool isInGitHubCodeBlock = false;
- for (Token token in tokens) {
- String comment = token.lexeme;
- // Skip GitHub code blocks.
- // https://help.github.com/articles/creating-and-highlighting-code-blocks/
- if (tokens.length != 1) {
- if (comment.indexOf('```') != -1) {
- isInGitHubCodeBlock = !isInGitHubCodeBlock;
- }
- if (isInGitHubCodeBlock) {
- continue;
- }
- }
- // Remove GitHub include code.
- comment = _removeGitHubInlineCode(comment);
- // Find references.
- int length = comment.length;
- List<List<int>> codeBlockRanges = _getCodeBlockRanges(comment);
- int leftIndex = comment.indexOf('[');
- while (leftIndex >= 0 && leftIndex + 1 < length) {
- List<int> range = _findRange(codeBlockRanges, leftIndex);
- if (range == null) {
- int nameOffset = token.offset + leftIndex + 1;
- int rightIndex = comment.indexOf(']', leftIndex);
- if (rightIndex >= 0) {
- int firstChar = comment.codeUnitAt(leftIndex + 1);
- if (firstChar != 0x27 && firstChar != 0x22) {
- if (_isLinkText(comment, rightIndex)) {
- // TODO(brianwilkerson) Handle the case where there's a library
- // URI in the link text.
- } else {
- CommentReference reference = _parseCommentReference(
- comment.substring(leftIndex + 1, rightIndex), nameOffset);
- if (reference != null) {
- references.add(reference);
- }
- }
- }
- } else {
- // terminating ']' is not typed yet
- int charAfterLeft = comment.codeUnitAt(leftIndex + 1);
- Token nameToken;
- if (Character.isLetterOrDigit(charAfterLeft)) {
- int nameEnd = StringUtilities.indexOfFirstNotLetterDigit(
- comment, leftIndex + 1);
- String name = comment.substring(leftIndex + 1, nameEnd);
- nameToken =
- new StringToken(TokenType.IDENTIFIER, name, nameOffset);
- } else {
- nameToken = new SyntheticStringToken(
- TokenType.IDENTIFIER, '', nameOffset);
- }
- nameToken.setNext(new Token.eof(nameToken.end));
- references.add(
- ast.commentReference(null, ast.simpleIdentifier(nameToken)));
- // next character
- rightIndex = leftIndex + 1;
- }
- leftIndex = comment.indexOf('[', rightIndex);
- } else {
- leftIndex = comment.indexOf('[', range[1]);
- }
- }
- }
- return references;
+ return comment;
}
/// Parse a documentation comment. Return the documentation comment that was
@@ -2922,48 +2666,23 @@
}
commentToken = commentToken.next;
}
- List<CommentReference> references = _parseCommentReferences(tokens);
- return tokens.isEmpty ? null : ast.documentationComment(tokens, references);
- }
-
- /// Remove any substrings in the given [comment] that represent in-line code
- /// in markdown.
- String _removeGitHubInlineCode(String comment) {
- int index = 0;
- while (true) {
- int beginIndex = comment.indexOf('`', index);
- if (beginIndex == -1) {
- break;
- }
- int endIndex = comment.indexOf('`', beginIndex + 1);
- if (endIndex == -1) {
- break;
- }
- comment = comment.substring(0, beginIndex + 1) +
- ' ' * (endIndex - beginIndex - 1) +
- comment.substring(endIndex);
- index = endIndex + 1;
+ if (tokens.isEmpty) {
+ return null;
}
- return comment;
+ int count = parser.parseCommentReferences(tokens.first);
+ // TODO(danrubel): If nulls were not added to the list, then this could
+ // be a fixed length list.
+ List<CommentReference> references = new List<CommentReference>()
+ ..length = count;
+ // popTypedList(...) returns `null` if count is zero,
+ // but ast.documentationComment(...) expects a non-null references list.
+ popTypedList(count, references);
+ // TODO(danrubel): Investigate preventing nulls from being added
+ // but for now, just remove them.
+ references.removeWhere((ref) => ref == null);
+ return ast.documentationComment(tokens, references);
}
- /// Return `true` if the given [token] has the given [type].
- bool _tokenMatches(Token token, TokenType type) => token.type == type;
-
- /// Return `true` if the given [token] is a valid identifier. Valid
- /// identifiers include built-in identifiers (pseudo-keywords).
- bool _tokenMatchesIdentifier(Token token) =>
- _tokenMatches(token, TokenType.IDENTIFIER) ||
- _tokenMatchesPseudoKeyword(token);
-
- /// Return `true` if the given [token] matches the given [keyword].
- bool _tokenMatchesKeyword(Token token, Keyword keyword) =>
- token.keyword == keyword;
-
- /// Return `true` if the given [token] matches a pseudo keyword.
- bool _tokenMatchesPseudoKeyword(Token token) =>
- token.keyword?.isBuiltInOrPseudo ?? false;
-
@override
void debugEvent(String name) {
// printEvent('AstBuilder: $name');
diff --git a/pkg/analyzer/test/generated/parser_fasta_test.dart b/pkg/analyzer/test/generated/parser_fasta_test.dart
index 3b7f341..df2e4c7 100644
--- a/pkg/analyzer/test/generated/parser_fasta_test.dart
+++ b/pkg/analyzer/test/generated/parser_fasta_test.dart
@@ -857,8 +857,16 @@
}
}
expect(tokens[tokens.length - 1].next, isNull);
- // TODO(danrubel): Implement this
- return null;
+ int count = fastaParser.parseCommentReferences(tokens[0]);
+ if (count == null) {
+ return null;
+ }
+ List<CommentReference> references = new List<CommentReference>(count);
+ // Since parseCommentReferences(...) returned non-null, then this method
+ // should return non-null in indicating that dartdoc comments were parsed.
+ // popTypedList(...) returns `null` if count is zero.
+ astBuilder.popTypedList(count, references);
+ return references;
}
@override
@@ -1013,97 +1021,7 @@
@reflectiveTest
class SimpleParserTest_Fasta extends FastaParserTestCase
- with SimpleParserTestMixin {
- @override
- @failingTest
- void test_parseCommentReferences_multiLine() {
- super.test_parseCommentReferences_multiLine();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_notClosed_noIdentifier() {
- super.test_parseCommentReferences_notClosed_noIdentifier();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_notClosed_withIdentifier() {
- super.test_parseCommentReferences_notClosed_withIdentifier();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_singleLine() {
- super.test_parseCommentReferences_singleLine();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipCodeBlock_4spaces_block() {
- super.test_parseCommentReferences_skipCodeBlock_4spaces_block();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipCodeBlock_4spaces_lines() {
- super.test_parseCommentReferences_skipCodeBlock_4spaces_lines();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipCodeBlock_bracketed() {
- super.test_parseCommentReferences_skipCodeBlock_bracketed();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipCodeBlock_gitHub() {
- super.test_parseCommentReferences_skipCodeBlock_gitHub();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipCodeBlock_gitHub_multiLine() {
- super.test_parseCommentReferences_skipCodeBlock_gitHub_multiLine();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipCodeBlock_gitHub_multiLine_lines() {
- super.test_parseCommentReferences_skipCodeBlock_gitHub_multiLine_lines();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipCodeBlock_gitHub_notTerminated() {
- super.test_parseCommentReferences_skipCodeBlock_gitHub_notTerminated();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipCodeBlock_spaces() {
- super.test_parseCommentReferences_skipCodeBlock_spaces();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipLinkDefinition() {
- super.test_parseCommentReferences_skipLinkDefinition();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipLinked() {
- super.test_parseCommentReferences_skipLinked();
- }
-
- @override
- @failingTest
- void test_parseCommentReferences_skipReferenceLink() {
- super.test_parseCommentReferences_skipReferenceLink();
- }
-}
+ with SimpleParserTestMixin {}
/**
* Tests of the fasta parser based on [StatementParserTestMixin].
diff --git a/pkg/analyzer/test/generated/parser_test.dart b/pkg/analyzer/test/generated/parser_test.dart
index cd6334f..f06a44b 100644
--- a/pkg/analyzer/test/generated/parser_test.dart
+++ b/pkg/analyzer/test/generated/parser_test.dart
@@ -13506,7 +13506,10 @@
@Annotation
/// This dartdoc comment is [included].
// a non dartdoc comment [inbetween]
-/// See [int] and [String]
+/// See [int] and [String] but `not [a]`
+/// ```
+/// This [code] block should be ignored
+/// ```
/// and [Object].
abstract class Foo {}
''');
@@ -13525,7 +13528,7 @@
expectReference(0, 'included', 86);
expectReference(1, 'int', 143);
expectReference(2, 'String', 153);
- expectReference(3, 'Object', 170);
+ expectReference(3, 'Object', 240);
}
void test_parseCommentReference_new_prefixed() {
diff --git a/pkg/front_end/lib/src/fasta/parser/forwarding_listener.dart b/pkg/front_end/lib/src/fasta/parser/forwarding_listener.dart
index 467e833..a471f62 100644
--- a/pkg/front_end/lib/src/fasta/parser/forwarding_listener.dart
+++ b/pkg/front_end/lib/src/fasta/parser/forwarding_listener.dart
@@ -923,6 +923,19 @@
}
@override
+ void handleCommentReference(
+ Token newKeyword, Token prefix, Token period, Token token) {
+ listener?.handleCommentReference(newKeyword, prefix, period, token);
+ }
+
+ @override
+ void handleCommentReferenceText(
+ Token commentToken, String referenceSource, int referenceOffset) {
+ listener?.handleCommentReferenceText(
+ commentToken, referenceSource, referenceOffset);
+ }
+
+ @override
void beginConditionalExpression(Token question) {
listener?.beginConditionalExpression(question);
}
@@ -1131,6 +1144,11 @@
}
@override
+ void handleNoCommentReference() {
+ listener?.handleNoCommentReference();
+ }
+
+ @override
void handleNoConstructorReferenceContinuationAfterTypeArguments(Token token) {
listener?.handleNoConstructorReferenceContinuationAfterTypeArguments(token);
}
diff --git a/pkg/front_end/lib/src/fasta/parser/listener.dart b/pkg/front_end/lib/src/fasta/parser/listener.dart
index 1f6dd5c..8c45723 100644
--- a/pkg/front_end/lib/src/fasta/parser/listener.dart
+++ b/pkg/front_end/lib/src/fasta/parser/listener.dart
@@ -1237,4 +1237,31 @@
/// has a type substitution comment /*=T*. So, the type that has been just
/// parsed should be discarded, and a new type should be parsed instead.
void discardTypeReplacedWithCommentTypeAssign() {}
+
+ /// A single comment reference in [commentToken] has been found where
+ /// where [referenceSource] is the text between the `[` and `]`
+ /// and [referenceOffset] is the character offset in the token stream.
+ ///
+ /// This event is generated by the parser when the parser's
+ /// `parseCommentReferences` method is called. For further processing,
+ /// a listener may scan the [referenceSource] and then pass the resulting
+ /// token stream to the parser's `parseOneCommentReference` method.
+ void handleCommentReferenceText(
+ Token commentToken, String referenceSource, int referenceOffset) {
+ logEvent("CommentReferenceText");
+ }
+
+ /// A single comment reference has been parsed.
+ /// * [newKeyword] may be null.
+ /// * [prefix] and [period] are either both tokens or both `null`.
+ /// * [token] can be an identifier or an operator.
+ ///
+ /// This event is generated by the parser when the parser's
+ /// `parseOneCommentReference` method is called.
+ void handleCommentReference(
+ Token newKeyword, Token prefix, Token period, Token token) {}
+
+ /// This event is generated by the parser when the parser's
+ /// `parseOneCommentReference` method is called.
+ void handleNoCommentReference() {}
}
diff --git a/pkg/front_end/lib/src/fasta/parser/parser.dart b/pkg/front_end/lib/src/fasta/parser/parser.dart
index daf4bbf4..c3071fd 100644
--- a/pkg/front_end/lib/src/fasta/parser/parser.dart
+++ b/pkg/front_end/lib/src/fasta/parser/parser.dart
@@ -15,6 +15,7 @@
ASSIGNMENT_PRECEDENCE,
BeginToken,
CASCADE_PRECEDENCE,
+ DocumentationCommentToken,
EQUALITY_PRECEDENCE,
Keyword,
POSTFIX_PRECEDENCE,
@@ -90,9 +91,12 @@
import 'util.dart'
show
- findPreviousNonZeroLengthToken,
findNonZeroLengthToken,
+ findPreviousNonZeroLengthToken,
+ isLetter,
+ isLetterOrDigit,
isOneOf,
+ isWhitespace,
optional;
/// An event generating parser of Dart programs. This parser expects all tokens
@@ -5828,6 +5832,268 @@
before.next = token;
return before;
}
+
+ /// Parse the comment references in a sequence of comment tokens
+ /// where [token] is the first in the sequence.
+ /// Return the number of comment references parsed
+ /// or `null` if there are no dartdoc comment tokens in the sequence.
+ int parseCommentReferences(Token token) {
+ if (token == null) {
+ return null;
+ }
+ Token singleLineDoc;
+ Token multiLineDoc;
+
+ // Find the first dartdoc token to parse
+ do {
+ String lexeme = token.lexeme;
+ if (lexeme.startsWith('/**')) {
+ singleLineDoc = null;
+ multiLineDoc = token;
+ } else if (lexeme.startsWith('///')) {
+ singleLineDoc ??= token;
+ multiLineDoc = null;
+ }
+ token = token.next;
+ } while (token != null);
+
+ // Parse the comment references
+ if (multiLineDoc != null) {
+ return parseReferencesInMultiLineComment(multiLineDoc);
+ } else if (singleLineDoc != null) {
+ return parseReferencesInSingleLineComments(singleLineDoc);
+ } else {
+ return null;
+ }
+ }
+
+ /// Parse the comment references in a multi-line comment token.
+ /// Return the number of comment references parsed.
+ int parseReferencesInMultiLineComment(Token multiLineDoc) {
+ String comment = multiLineDoc.lexeme;
+ assert(comment.startsWith('/**'));
+ int count = 0;
+ int length = comment.length;
+ int start = 3;
+ bool inCodeBlock = false;
+ int codeBlock = comment.indexOf('```', 3);
+ if (codeBlock == -1) {
+ codeBlock = length;
+ }
+ while (start < length) {
+ if (isWhitespace(comment.codeUnitAt(start))) {
+ ++start;
+ continue;
+ }
+ int end = comment.indexOf('\n', start);
+ if (end == -1) {
+ end = length;
+ }
+ if (codeBlock < end) {
+ inCodeBlock = !inCodeBlock;
+ codeBlock = comment.indexOf('```', end);
+ if (codeBlock == -1) {
+ codeBlock = length;
+ }
+ }
+ if (!inCodeBlock && !comment.startsWith('* ', start)) {
+ count += parseCommentReferencesInText(multiLineDoc, start, end);
+ }
+ start = end + 1;
+ }
+ return count;
+ }
+
+ /// Parse the comment references in a sequence of single line comment tokens
+ /// where [token] is the first comment token in the sequence.
+ /// Return the number of comment references parsed.
+ int parseReferencesInSingleLineComments(Token token) {
+ int count = 0;
+ bool inCodeBlock = false;
+ while (token != null && !token.isEof) {
+ String comment = token.lexeme;
+ if (comment.startsWith('///')) {
+ if (comment.indexOf('```', 3) != -1) {
+ inCodeBlock = !inCodeBlock;
+ }
+ if (!inCodeBlock && !comment.startsWith('/// ')) {
+ count += parseCommentReferencesInText(token, 3, comment.length);
+ }
+ }
+ token = token.next;
+ }
+ return count;
+ }
+
+ /// Parse the comment references in the text between [start] inclusive
+ /// and [end] exclusive. Return a count indicating how many were parsed.
+ int parseCommentReferencesInText(Token commentToken, int start, int end) {
+ String comment = commentToken.lexeme;
+ int count = 0;
+ int index = start;
+ while (index < end) {
+ int ch = comment.codeUnitAt(index);
+ if (ch == 0x5B /* `[` */) {
+ ++index;
+ if (index < end && comment.codeUnitAt(index) == 0x3A /* `:` */) {
+ // Skip old-style code block.
+ index = comment.indexOf(':]', index + 1) + 1;
+ if (index == 0 || index > end) {
+ break;
+ }
+ } else {
+ int referenceStart = index;
+ index = comment.indexOf(']', index);
+ if (index == -1 || index >= end) {
+ // Recovery: terminating ']' is not typed yet.
+ index = findReferenceEnd(comment, referenceStart, end);
+ }
+ if (ch != 0x27 /* `'` */ && ch != 0x22 /* `"` */) {
+ if (isLinkText(comment, index)) {
+ // TODO(brianwilkerson) Handle the case where there's a library
+ // URI in the link text.
+ } else {
+ listener.handleCommentReferenceText(
+ commentToken,
+ comment.substring(referenceStart, index),
+ commentToken.charOffset + referenceStart);
+ ++count;
+ }
+ }
+ }
+ } else if (ch == 0x60 /* '`' */) {
+ // Skip inline code block if there is both starting '`' and ending '`'
+ int endCodeBlock = comment.indexOf('`', index + 1);
+ if (endCodeBlock != -1 && endCodeBlock < end) {
+ index = endCodeBlock;
+ }
+ }
+ ++index;
+ }
+ return count;
+ }
+
+ /// Given a comment reference without a closing `]`,
+ /// search for a possible place where `]` should be.
+ int findReferenceEnd(String comment, int index, int end) {
+ // Find the end of the identifier if there is one
+ if (index >= end || !isLetter(comment.codeUnitAt(index))) {
+ return index;
+ }
+ while (index < end && isLetterOrDigit(comment.codeUnitAt(index))) {
+ ++index;
+ }
+
+ // Check for a trailing `.`
+ if (index >= end || comment.codeUnitAt(index) != 0x2E /* `.` */) {
+ return index;
+ }
+ ++index;
+
+ // Find end of the identifier after the `.`
+ if (index >= end || !isLetter(comment.codeUnitAt(index))) {
+ return index;
+ }
+ ++index;
+ while (index < end && isLetterOrDigit(comment.codeUnitAt(index))) {
+ ++index;
+ }
+ return index;
+ }
+
+ /// Parse the tokens in a single comment reference and generate either a
+ /// `handleCommentReference` or `handleNoCommentReference` event.
+ ///
+ /// This is typically called from the listener's
+ /// `handleCommentReferenceText` method.
+ void parseOneCommentReference(
+ DocumentationCommentToken commentToken, Token token) {
+ Token begin = token;
+ Token newKeyword = null;
+ if (optional('new', token)) {
+ newKeyword = token;
+ token = token.next;
+ }
+ Token prefix, period;
+ if (token.isIdentifier && optional('.', token.next)) {
+ prefix = token;
+ period = token.next;
+ token = period.next;
+ }
+ if (token.isEof) {
+ // Recovery: Insert a synthetic identifier for code completion
+ token = rewriter.insertSyntheticIdentifier(
+ period ?? newKeyword ?? syntheticPreviousToken(token));
+ if (begin == token.next) {
+ begin = token;
+ }
+ }
+ Token operatorKeyword = null;
+ if (optional('operator', token)) {
+ operatorKeyword = token;
+ token = token.next;
+ }
+ if (token.isUserDefinableOperator) {
+ if (token.next.isEof) {
+ listener.handleCommentReference(newKeyword, prefix, period, token);
+ commentToken.references.add(begin);
+ return;
+ }
+ } else {
+ token = operatorKeyword ?? token;
+ if (token.next.isEof) {
+ if (token.isIdentifier) {
+ listener.handleCommentReference(newKeyword, prefix, period, token);
+ commentToken.references.add(begin);
+ return;
+ }
+ Keyword keyword = token.keyword;
+ if (newKeyword == null &&
+ prefix == null &&
+ (keyword == Keyword.THIS ||
+ keyword == Keyword.NULL ||
+ keyword == Keyword.TRUE ||
+ keyword == Keyword.FALSE)) {
+ // TODO(brianwilkerson) If we want to support this we will need to
+ // extend the definition of CommentReference to take an expression
+ // rather than an identifier. For now we just ignore it to reduce the
+ // number of errors produced, but that's probably not a valid long
+ // term approach.
+ }
+ }
+ }
+ listener.handleNoCommentReference();
+ }
+
+ /// Given that we have just found bracketed text within the given [comment],
+ /// look to see whether that text is (a) followed by a parenthesized link
+ /// address, (b) followed by a colon, or (c) followed by optional whitespace
+ /// and another square bracket. The [rightIndex] is the index of the right
+ /// bracket. Return `true` if the bracketed text is followed by a link
+ /// address.
+ ///
+ /// This method uses the syntax described by the
+ /// <a href="http://daringfireball.net/projects/markdown/syntax">markdown</a>
+ /// project.
+ bool isLinkText(String comment, int rightIndex) {
+ int length = comment.length;
+ int index = rightIndex + 1;
+ if (index >= length) {
+ return false;
+ }
+ int ch = comment.codeUnitAt(index);
+ if (ch == 0x28 || ch == 0x3A) {
+ return true;
+ }
+ while (isWhitespace(ch)) {
+ index = index + 1;
+ if (index >= length) {
+ return false;
+ }
+ ch = comment.codeUnitAt(index);
+ }
+ return ch == 0x5B;
+ }
}
// TODO(ahe): Remove when analyzer supports generalized function syntax.
diff --git a/pkg/front_end/lib/src/fasta/parser/util.dart b/pkg/front_end/lib/src/fasta/parser/util.dart
index ab00611..4329810 100644
--- a/pkg/front_end/lib/src/fasta/parser/util.dart
+++ b/pkg/front_end/lib/src/fasta/parser/util.dart
@@ -62,6 +62,14 @@
return token == null ? TreeNode.noOffset : token.offset;
}
+bool isDigit(int c) => c >= 0x30 && c <= 0x39;
+
+bool isLetter(int c) => c >= 0x41 && c <= 0x5A || c >= 0x61 && c <= 0x7A;
+
+bool isLetterOrDigit(int c) => isLetter(c) || isDigit(c);
+
+bool isWhitespace(int c) => c == 0x20 || c == 0xA || c == 0xD || c == 0x9;
+
/// Return true if the given token matches one of the given values.
bool isOneOf(Token token, Iterable<String> values) {
for (String tokenValue in values) {
diff --git a/pkg/front_end/lib/src/fasta/source/stack_listener.dart b/pkg/front_end/lib/src/fasta/source/stack_listener.dart
index df55a69..e7f2eb4a 100644
--- a/pkg/front_end/lib/src/fasta/source/stack_listener.dart
+++ b/pkg/front_end/lib/src/fasta/source/stack_listener.dart
@@ -32,6 +32,7 @@
BreakTarget,
CascadeReceiver,
Combinators,
+ CommentReference,
Comments,
ConditionalUris,
ConditionallySelectedImport,
