Add TokenStreamGhostWriter for parser lookahead
This adds a new TokenStreamRewriter which does not modify the
existing token stream but instead sequences tokens such that the
new tokens lead into the existing token stream.
This is useful when using the parseExpression method
to lookahead and find the next token after the expression.
Change-Id: Id51e015f4381af65220530d07042e306b939baa3
Reviewed-on: https://dart-review.googlesource.com/c/87622
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>
diff --git a/pkg/front_end/lib/src/fasta/parser/token_stream_rewriter.dart b/pkg/front_end/lib/src/fasta/parser/token_stream_rewriter.dart
index 86170d4..9a8b367 100644
--- a/pkg/front_end/lib/src/fasta/parser/token_stream_rewriter.dart
+++ b/pkg/front_end/lib/src/fasta/parser/token_stream_rewriter.dart
@@ -37,9 +37,6 @@
// }
//
- /// Initialize a newly created re-writer.
- TokenStreamRewriter();
-
/// Insert a synthetic open and close parenthesis and return the new synthetic
/// open parenthesis. If [insertIdentifier] is true, then a synthetic
/// identifier is included between the open and close parenthesis.
@@ -140,3 +137,70 @@
return current;
}
}
+
+/// Provides the capability of adding tokens that lead into a token stream
+/// without modifying the original token stream and not setting the any token's
+/// `previous` field.
+class TokenStreamGhostWriter implements TokenStreamRewriter {
+ @override
+ Token insertParens(Token token, bool includeIdentifier) {
+ Token next = token.next;
+ int offset = next.charOffset;
+ BeginToken leftParen =
+ next = new SyntheticBeginToken(TokenType.OPEN_PAREN, offset);
+ if (includeIdentifier) {
+ Token identifier =
+ new SyntheticStringToken(TokenType.IDENTIFIER, '', offset, 0);
+ next.next = identifier;
+ next = identifier;
+ }
+ Token rightParen = new SyntheticToken(TokenType.CLOSE_PAREN, offset);
+ next.next = rightParen;
+ rightParen.next = token.next;
+
+ return leftParen;
+ }
+
+ /// Insert a synthetic identifier after [token] and return the new identifier.
+ Token insertSyntheticIdentifier(Token token) {
+ return insertToken(
+ token,
+ new SyntheticStringToken(
+ TokenType.IDENTIFIER, '', token.next.charOffset, 0));
+ }
+
+ @override
+ Token insertToken(Token token, Token newToken) {
+ newToken.next = token.next;
+ return newToken;
+ }
+
+ @override
+ Token moveSynthetic(Token token, Token endGroup) {
+ Token newEndGroup =
+ new SyntheticToken(endGroup.type, token.next.charOffset);
+ newEndGroup.next = token.next;
+ return newEndGroup;
+ }
+
+ @override
+ Token replaceTokenFollowing(Token previousToken, Token replacementToken) {
+ Token replacedToken = previousToken.next;
+
+ (replacementToken as SimpleToken).precedingComments =
+ replacedToken.precedingComments;
+
+ _lastTokenInChain(replacementToken).next = replacedToken.next;
+ return replacementToken;
+ }
+
+ /// Given the [firstToken] in a chain of tokens to be inserted, return the
+ /// last token in the chain.
+ Token _lastTokenInChain(Token firstToken) {
+ Token current = firstToken;
+ while (current.next != null && current.next.type != TokenType.EOF) {
+ current = current.next;
+ }
+ return current;
+ }
+}
diff --git a/pkg/front_end/test/fasta/parser/token_stream_rewriter_test.dart b/pkg/front_end/test/fasta/parser/token_stream_rewriter_test.dart
index f47b461..a4f4aa5 100644
--- a/pkg/front_end/test/fasta/parser/token_stream_rewriter_test.dart
+++ b/pkg/front_end/test/fasta/parser/token_stream_rewriter_test.dart
@@ -12,6 +12,7 @@
main() {
defineReflectiveSuite(() {
+ defineReflectiveTests(TokenStreamGhostWriterTest);
defineReflectiveTests(TokenStreamRewriterTest_NoPrevious);
defineReflectiveTests(TokenStreamRewriterTest_UsingPrevious);
});
@@ -22,6 +23,71 @@
/// Indicates whether the tests should set up [Token.previous].
bool get setPrevious;
+ void test_insertParens() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var eof = _link([a, b]);
+ var rewriter = new TokenStreamRewriter();
+ var openParen = rewriter.insertParens(a, false);
+ var closeParen = openParen.next;
+
+ expect(openParen.lexeme, '(');
+ expect(closeParen.lexeme, ')');
+
+ expect(a.next, same(openParen));
+ expect(openParen.next, same(closeParen));
+ expect(closeParen.next, same(b));
+ expect(b.next, same(eof));
+
+ expect(b.previous, same(closeParen));
+ expect(closeParen.previous, same(openParen));
+ expect(openParen.previous, same(a));
+ }
+
+ void test_insertParensWithIdentifier() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var eof = _link([a, b]);
+ var rewriter = new TokenStreamRewriter();
+ var openParen = rewriter.insertParens(a, true);
+ var identifier = openParen.next;
+ var closeParen = identifier.next;
+
+ expect(openParen.lexeme, '(');
+ expect(identifier.lexeme, '');
+ expect(identifier.isSynthetic, isTrue);
+ expect(closeParen.lexeme, ')');
+
+ expect(a.next, same(openParen));
+ expect(openParen.next, same(identifier));
+ expect(identifier.next, same(closeParen));
+ expect(closeParen.next, same(b));
+ expect(b.next, same(eof));
+
+ expect(b.previous, same(closeParen));
+ expect(closeParen.previous, same(identifier));
+ expect(identifier.previous, same(openParen));
+ expect(openParen.previous, same(a));
+ }
+
+ void test_insertSyntheticIdentifier() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var eof = _link([a, b]);
+ var rewriter = new TokenStreamRewriter();
+ var identifier = rewriter.insertSyntheticIdentifier(a);
+
+ expect(identifier.lexeme, '');
+ expect(identifier.isSynthetic, isTrue);
+
+ expect(a.next, same(identifier));
+ expect(identifier.next, same(b));
+ expect(b.next, same(eof));
+
+ expect(b.previous, same(identifier));
+ expect(identifier.previous, same(a));
+ }
+
void test_insertToken_end() {
var a = _makeToken(0, 'a');
var b = _makeToken(1, 'b');
@@ -142,9 +208,6 @@
/// finding previous tokens.
@reflectiveTest
class TokenStreamRewriterTest_NoPrevious extends TokenStreamRewriterTest {
- // These tests are failing because the re-writer currently depends on the
- // previous pointer.
-
@override
bool get setPrevious => false;
}
@@ -160,3 +223,201 @@
@override
bool get setPrevious => true;
}
+
+@reflectiveTest
+class TokenStreamGhostWriterTest extends TokenStreamRewriterTest {
+ @override
+ bool get setPrevious => false;
+
+ void test_insertParens() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var eof = _link([a, b]);
+ var rewriter = new TokenStreamGhostWriter();
+ var openParen = rewriter.insertParens(a, false);
+ var closeParen = openParen.next;
+
+ expect(openParen.lexeme, '(');
+ expect(closeParen.lexeme, ')');
+
+ expect(a.next, same(b));
+ expect(openParen.next, same(closeParen));
+ expect(closeParen.next, same(b));
+ expect(b.next, same(eof));
+
+ expect(b.previous, isNull);
+ expect(closeParen.previous, isNull);
+ expect(openParen.previous, isNull);
+ }
+
+ void test_insertParensWithIdentifier() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var eof = _link([a, b]);
+ var rewriter = new TokenStreamGhostWriter();
+ var openParen = rewriter.insertParens(a, true);
+ var identifier = openParen.next;
+ var closeParen = identifier.next;
+
+ expect(openParen.lexeme, '(');
+ expect(identifier.lexeme, '');
+ expect(identifier.isSynthetic, isTrue);
+ expect(closeParen.lexeme, ')');
+
+ expect(a.next, same(b));
+ expect(openParen.next, same(identifier));
+ expect(identifier.next, same(closeParen));
+ expect(closeParen.next, same(b));
+ expect(b.next, same(eof));
+
+ expect(b.previous, isNull);
+ expect(closeParen.previous, isNull);
+ expect(identifier.previous, isNull);
+ expect(openParen.previous, isNull);
+ }
+
+ void test_insertSyntheticIdentifier() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var eof = _link([a, b]);
+ var rewriter = new TokenStreamGhostWriter();
+ var identifier = rewriter.insertSyntheticIdentifier(a);
+
+ expect(identifier.lexeme, '');
+ expect(identifier.isSynthetic, isTrue);
+
+ expect(a.next, same(b));
+ expect(identifier.next, same(b));
+ expect(b.next, same(eof));
+
+ expect(b.previous, isNull);
+ expect(identifier.previous, isNull);
+ }
+
+ void test_insertToken_end() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var eof = _link([a]);
+ var rewriter = new TokenStreamGhostWriter();
+
+ expect(rewriter.insertToken(a, b), same(b));
+ expect(a.next, same(eof));
+ expect(b.next, same(eof));
+
+ expect(eof.previous, isNull);
+ expect(b.previous, isNull);
+ }
+
+ void test_insertToken_middle() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var c = _makeToken(2, 'c');
+ _link([a, c]);
+
+ var rewriter = new TokenStreamGhostWriter();
+ rewriter.insertToken(a, b);
+ expect(a.next, same(c));
+ expect(b.next, same(c));
+
+ expect(a.previous, isNull);
+ expect(b.previous, isNull);
+ expect(c.previous, isNull);
+ }
+
+ void test_insertToken_second_insertion_earlier_in_stream() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var c = _makeToken(2, 'c');
+ var d = _makeToken(3, 'd');
+ var e = _makeToken(4, 'e');
+ _link([a, c, e]);
+ var rewriter = new TokenStreamGhostWriter();
+
+ rewriter.insertToken(c, d);
+ expect(c.next, same(e));
+ expect(d.next, same(e));
+
+ // The next call to rewriter should be able to find the insertion point
+ // even though it is before the insertion point used above.
+ rewriter.insertToken(a, b);
+ expect(a.next, same(c));
+ expect(b.next, same(c));
+
+ expect(a.previous, isNull);
+ expect(b.previous, isNull);
+ expect(c.previous, isNull);
+ expect(d.previous, isNull);
+ expect(e.previous, isNull);
+ }
+
+ void test_moveSynthetic() {
+ ScannerResult scanResult = scanString('Foo(bar; baz=0;');
+ expect(scanResult.hasErrors, isTrue);
+ Token open = scanResult.tokens.next.next;
+ expect(open.lexeme, '(');
+ Token semicolon = open.next.next;
+ expect(semicolon.lexeme, ';');
+ Token close = open.endGroup;
+ expect(close.isSynthetic, isTrue);
+ expect(close.next.isEof, isTrue);
+ Token semicolon2 = close.previous;
+ expect(semicolon2.lexeme, ';');
+ var rewriter = new TokenStreamGhostWriter();
+
+ Token newClose = rewriter.moveSynthetic(open.next, close);
+ expect(newClose, isNot(same(close)));
+ expect(newClose.next, same(semicolon));
+ expect(open.endGroup, close);
+ expect(open.next.next, semicolon);
+ expect(close.next.isEof, isTrue);
+
+ expect(newClose.previous, isNull);
+ expect(close.next.previous, close);
+ expect(close.previous, semicolon2);
+ }
+
+ void test_replaceTokenFollowing_multiple() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var c = _makeToken(2, 'c');
+ var d = _makeToken(3, 'd');
+ var e = _makeToken(4, 'e');
+ var f = _makeToken(5, 'f');
+ _link([a, b, e, f]);
+ _link([c, d]);
+ var rewriter = new TokenStreamGhostWriter();
+ Token result = rewriter.replaceTokenFollowing(b, c);
+
+ expect(result, same(c));
+ expect(a.next, same(b));
+ expect(b.next, same(e));
+ expect(e.next, same(f));
+ expect(c.next, same(d));
+ expect(d.next, same(f));
+
+ expect(a.previous, isNull);
+ expect(b.previous, isNull);
+ expect(c.previous, isNull);
+ expect(d.previous, isNull);
+ expect(e.previous, isNull);
+ }
+
+ void test_replaceTokenFollowing_single() {
+ var a = _makeToken(0, 'a');
+ var b = _makeToken(1, 'b');
+ var c = _makeToken(2, 'c');
+ var d = _makeToken(3, 'd');
+ _link([a, b, d]);
+ var rewriter = new TokenStreamGhostWriter();
+ Token result = rewriter.replaceTokenFollowing(a, c);
+
+ expect(result, same(c));
+ expect(a.next, same(b));
+ expect(b.next, same(d));
+ expect(c.next, same(d));
+
+ expect(a.previous, isNull);
+ expect(b.previous, isNull);
+ expect(c.previous, isNull);
+ }
+}
