Improve fasta parser expression recovery
This improves the parser's ability to distinguish between an expression
and the start of a new statement.
Change-Id: I07871dc0e2e6f81996bfc99a53bd3dede3db21ae
Reviewed-on: https://dart-review.googlesource.com/62180
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>
diff --git a/pkg/analyzer/test/src/fasta/recovery/partial_code/assert_statement_test.dart b/pkg/analyzer/test/src/fasta/recovery/partial_code/assert_statement_test.dart
index f80505e..decfa5e 100644
--- a/pkg/analyzer/test/src/fasta/recovery/partial_code/assert_statement_test.dart
+++ b/pkg/analyzer/test/src/fasta/recovery/partial_code/assert_statement_test.dart
@@ -31,7 +31,14 @@
ParserErrorCode.EXPECTED_TOKEN
],
"assert (_s_);",
- failing: allExceptEof),
+ failing: [
+ 'assert',
+ 'block',
+ 'labeled',
+ 'localFunctionNonVoid',
+ 'localFunctionVoid',
+ 'return'
+ ]),
new TestDescriptor(
'condition',
'assert (a',
diff --git a/pkg/analyzer/test/src/fasta/recovery/partial_code/field_declaration_test.dart b/pkg/analyzer/test/src/fasta/recovery/partial_code/field_declaration_test.dart
index a4ddf6b..af989fd 100644
--- a/pkg/analyzer/test/src/fasta/recovery/partial_code/field_declaration_test.dart
+++ b/pkg/analyzer/test/src/fasta/recovery/partial_code/field_declaration_test.dart
@@ -56,7 +56,13 @@
'const f =',
[ParserErrorCode.MISSING_IDENTIFIER, ParserErrorCode.EXPECTED_TOKEN],
'const f = _s_;',
- failing: allExceptEof,
+ failing: [
+ 'fieldConst',
+ 'methodNonVoid',
+ 'methodVoid',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'const_initializer',
@@ -86,7 +92,13 @@
'final f =',
[ParserErrorCode.MISSING_IDENTIFIER, ParserErrorCode.EXPECTED_TOKEN],
'final f = _s_;',
- failing: allExceptEof,
+ failing: [
+ 'fieldConst',
+ 'methodNonVoid',
+ 'methodVoid',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'final_initializer',
@@ -116,7 +128,13 @@
'var f =',
[ParserErrorCode.MISSING_IDENTIFIER, ParserErrorCode.EXPECTED_TOKEN],
'var f = _s_;',
- failing: allExceptEof,
+ failing: [
+ 'fieldConst',
+ 'methodNonVoid',
+ 'methodVoid',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'var_initializer',
@@ -148,7 +166,13 @@
'A f =',
[ParserErrorCode.MISSING_IDENTIFIER, ParserErrorCode.EXPECTED_TOKEN],
'A f = _s_;',
- failing: allExceptEof,
+ failing: [
+ 'fieldConst',
+ 'methodNonVoid',
+ 'methodVoid',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'type_initializer',
@@ -187,7 +211,13 @@
'static const f =',
[ParserErrorCode.MISSING_IDENTIFIER, ParserErrorCode.EXPECTED_TOKEN],
'static const f = _s_;',
- failing: allExceptEof,
+ failing: [
+ 'fieldConst',
+ 'methodNonVoid',
+ 'methodVoid',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'static_const_initializer',
@@ -217,7 +247,13 @@
'static final f =',
[ParserErrorCode.MISSING_IDENTIFIER, ParserErrorCode.EXPECTED_TOKEN],
'static final f = _s_;',
- failing: allExceptEof,
+ failing: [
+ 'fieldConst',
+ 'methodNonVoid',
+ 'methodVoid',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'static_final_initializer',
@@ -247,7 +283,13 @@
'static var f =',
[ParserErrorCode.MISSING_IDENTIFIER, ParserErrorCode.EXPECTED_TOKEN],
'static var f = _s_;',
- failing: allExceptEof,
+ failing: [
+ 'fieldConst',
+ 'methodNonVoid',
+ 'methodVoid',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'static_var_initializer',
@@ -279,7 +321,13 @@
'static A f =',
[ParserErrorCode.MISSING_IDENTIFIER, ParserErrorCode.EXPECTED_TOKEN],
'static A f = _s_;',
- failing: allExceptEof,
+ failing: [
+ 'fieldConst',
+ 'methodNonVoid',
+ 'methodVoid',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'static_type_initializer',
diff --git a/pkg/analyzer/test/src/fasta/recovery/partial_code/local_variable_test.dart b/pkg/analyzer/test/src/fasta/recovery/partial_code/local_variable_test.dart
index 388eec1..dc875ff 100644
--- a/pkg/analyzer/test/src/fasta/recovery/partial_code/local_variable_test.dart
+++ b/pkg/analyzer/test/src/fasta/recovery/partial_code/local_variable_test.dart
@@ -139,7 +139,14 @@
ParserErrorCode.EXPECTED_TOKEN
],
"var a = _s_;",
- failing: allExceptEof),
+ failing: [
+ 'block',
+ 'assert',
+ 'labeled',
+ 'localFunctionNonVoid',
+ 'localFunctionVoid',
+ 'return'
+ ]),
new TestDescriptor('varNameEqualsExpression', 'var a = b',
[ParserErrorCode.EXPECTED_TOKEN], "var a = b;"),
],
diff --git a/pkg/analyzer/test/src/fasta/recovery/partial_code/top_level_variable_test.dart b/pkg/analyzer/test/src/fasta/recovery/partial_code/top_level_variable_test.dart
index bb3264c..76e6f1c 100644
--- a/pkg/analyzer/test/src/fasta/recovery/partial_code/top_level_variable_test.dart
+++ b/pkg/analyzer/test/src/fasta/recovery/partial_code/top_level_variable_test.dart
@@ -214,7 +214,15 @@
ParserErrorCode.EXPECTED_TOKEN
],
"var a = _s_;",
- failing: allExceptEof,
+ failing: [
+ 'class',
+ 'typedef',
+ 'functionVoid',
+ 'functionNonVoid',
+ 'const',
+ 'getter',
+ 'setter'
+ ],
),
new TestDescriptor(
'varNameEqualsExpression',
diff --git a/pkg/analyzer/test/src/fasta/recovery/partial_code/yield_statement_test.dart b/pkg/analyzer/test/src/fasta/recovery/partial_code/yield_statement_test.dart
index 0234d1f..bc15d07 100644
--- a/pkg/analyzer/test/src/fasta/recovery/partial_code/yield_statement_test.dart
+++ b/pkg/analyzer/test/src/fasta/recovery/partial_code/yield_statement_test.dart
@@ -26,19 +26,10 @@
failing: [
'assert',
'block',
- 'break',
- 'continue',
- 'do',
- 'if',
- 'for',
'labeled',
'localFunctionNonVoid',
'localFunctionVoid',
- 'localVariable',
- 'switch',
- 'try',
'return',
- 'while'
]),
new TestDescriptor('expression', 'yield a',
[ParserErrorCode.EXPECTED_TOKEN], "yield a;"),
@@ -53,19 +44,10 @@
failing: [
'assert',
'block',
- 'break',
- 'continue',
- 'do',
- 'if',
- 'for',
'labeled',
'localFunctionNonVoid',
'localFunctionVoid',
- 'localVariable',
- 'switch',
- 'try',
'return',
- 'while'
]),
new TestDescriptor('star_expression', 'yield * a',
[ParserErrorCode.EXPECTED_TOKEN], "yield * a;"),
diff --git a/pkg/front_end/lib/src/fasta/parser/identifier_context.dart b/pkg/front_end/lib/src/fasta/parser/identifier_context.dart
index 601835b..43c121c 100644
--- a/pkg/front_end/lib/src/fasta/parser/identifier_context.dart
+++ b/pkg/front_end/lib/src/fasta/parser/identifier_context.dart
@@ -2,7 +2,7 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
-import '../../scanner/token.dart' show Token;
+import '../../scanner/token.dart' show Token, TokenType;
import '../fasta_codes.dart' show Message, Template, templateExpectedIdentifier;
@@ -12,6 +12,8 @@
import 'parser.dart' show Parser;
+import 'util.dart' show isOneOfOrEof, optional;
+
/// Information about the parser state that is passed to the listener at the
/// time an identifier is encountered. It is also used by the parser for error
/// recovery when a recovery template is defined.
@@ -269,5 +271,45 @@
}
}
+/// Return `true` if the given [token] should be treated like the start of
+/// an expression for the purposes of recovery.
+bool looksLikeExpressionStart(Token next) =>
+ next.isIdentifier ||
+ next.isKeyword && !looksLikeStatementStart(next) ||
+ next.type == TokenType.DOUBLE ||
+ next.type == TokenType.HASH ||
+ next.type == TokenType.HEXADECIMAL ||
+ next.type == TokenType.IDENTIFIER ||
+ next.type == TokenType.INT ||
+ next.type == TokenType.STRING ||
+ optional('{', next) ||
+ optional('(', next) ||
+ optional('[', next) ||
+ optional('[]', next) ||
+ optional('<', next) ||
+ optional('!', next) ||
+ optional('-', next) ||
+ optional('~', next) ||
+ optional('++', next) ||
+ optional('--', next);
+
+/// Return `true` if the given [token] should be treated like the start of
+/// a new statement for the purposes of recovery.
+bool looksLikeStatementStart(Token token) => isOneOfOrEof(token, const [
+ 'assert',
+ 'break',
+ 'continue',
+ 'do',
+ 'final',
+ 'for',
+ 'if',
+ 'return',
+ 'switch',
+ 'try',
+ 'var',
+ 'void',
+ 'while'
+ ]);
+
// TODO(ahe): Remove when analyzer supports generalized function syntax.
typedef _MessageWithArgument<T> = Message Function(T);
diff --git a/pkg/front_end/lib/src/fasta/parser/identifier_context_impl.dart b/pkg/front_end/lib/src/fasta/parser/identifier_context_impl.dart
index e8cab59..c79c8a4 100644
--- a/pkg/front_end/lib/src/fasta/parser/identifier_context_impl.dart
+++ b/pkg/front_end/lib/src/fasta/parser/identifier_context_impl.dart
@@ -31,7 +31,7 @@
// Recovery
parser.reportRecoverableError(identifier, fasta.messageCatchSyntax);
- if (looksLikeStartOfNextStatement(identifier) ||
+ if (looksLikeStatementStart(identifier) ||
isOneOfOrEof(identifier, const [',', ')'])) {
return parser.rewriter.insertSyntheticIdentifier(token);
} else if (!identifier.isKeywordOrIdentifier) {
@@ -291,16 +291,18 @@
// Recovery
parser.reportRecoverableErrorWithToken(
identifier, fasta.templateExpectedIdentifier);
- if (identifier.isKeywordOrIdentifier) {
- if (!isOneOfOrEof(identifier, const ['as', 'is'])) {
- return identifier;
+ if (!looksLikeStatementStart(identifier)) {
+ if (identifier.isKeywordOrIdentifier) {
+ if (!isOneOfOrEof(identifier, const ['as', 'is'])) {
+ return identifier;
+ }
+ } else if (!identifier.isOperator &&
+ !isOneOfOrEof(identifier,
+ const ['.', ',', '(', ')', '[', ']', '}', '?', ':', ';'])) {
+ // When in doubt, consume the token to ensure we make progress
+ token = identifier;
+ identifier = token.next;
}
- } else if (!identifier.isOperator &&
- !isOneOfOrEof(identifier,
- const ['.', ',', '(', ')', '[', ']', '}', '?', ':', ';'])) {
- // When in doubt, consume the token to ensure we make progress
- token = identifier;
- identifier = token.next;
}
// Insert a synthetic identifier to satisfy listeners.
return parser.rewriter.insertSyntheticIdentifier(token);
@@ -377,7 +379,7 @@
// Recovery
const followingValues = const [':', '=', ',', '(', ')', '[', ']', '{', '}'];
if (looksLikeStartOfNextClassMember(identifier) ||
- looksLikeStartOfNextStatement(identifier) ||
+ looksLikeStatementStart(identifier) ||
isOneOfOrEof(identifier, followingValues)) {
identifier = parser.insertSyntheticIdentifier(token, this,
message: fasta.templateExpectedIdentifier.withArguments(identifier));
@@ -474,7 +476,7 @@
// Recovery
if (isOneOfOrEof(identifier, const ['.', '(', '{', '=>']) ||
- looksLikeStartOfNextStatement(identifier)) {
+ looksLikeStatementStart(identifier)) {
identifier = parser.insertSyntheticIdentifier(token, this,
message: fasta.templateExpectedIdentifier.withArguments(identifier));
} else {
@@ -506,7 +508,7 @@
// Recovery
if (isOneOfOrEof(identifier, const [':']) ||
- looksLikeStartOfNextStatement(identifier)) {
+ looksLikeStatementStart(identifier)) {
identifier = parser.insertSyntheticIdentifier(token, this,
message: fasta.templateExpectedIdentifier.withArguments(identifier));
} else {
@@ -622,7 +624,7 @@
// Recovery
if (isOneOfOrEof(identifier, const [';', '=', ',', '{', '}']) ||
- looksLikeStartOfNextStatement(identifier)) {
+ looksLikeStatementStart(identifier)) {
identifier = parser.insertSyntheticIdentifier(token, this,
message: fasta.templateExpectedIdentifier.withArguments(identifier));
} else {
@@ -664,7 +666,7 @@
if (isOneOfOrEof(identifier, const ['{', '}', '(', ')', ']']) ||
looksLikeStartOfNextTopLevelDeclaration(identifier) ||
looksLikeStartOfNextClassMember(identifier) ||
- looksLikeStartOfNextStatement(identifier)) {
+ looksLikeStatementStart(identifier)) {
identifier = parser.insertSyntheticIdentifier(token, this,
message: fasta.templateExpectedIdentifier.withArguments(identifier));
} else {
@@ -908,7 +910,7 @@
const followingValues = const ['<', '>', ';', '}', 'extends', 'super'];
if (looksLikeStartOfNextTopLevelDeclaration(identifier) ||
looksLikeStartOfNextClassMember(identifier) ||
- looksLikeStartOfNextStatement(identifier) ||
+ looksLikeStatementStart(identifier) ||
isOneOfOrEof(identifier, followingValues)) {
parser.reportRecoverableErrorWithToken(
identifier, fasta.templateExpectedIdentifier);
@@ -944,23 +946,6 @@
bool looksLikeStartOfNextClassMember(Token token) =>
token.isModifier || isOneOfOrEof(token, const ['get', 'set', 'void']);
-bool looksLikeStartOfNextStatement(Token token) => isOneOfOrEof(token, const [
- 'assert',
- 'break',
- 'const',
- 'continue',
- 'do',
- 'final',
- 'for',
- 'if',
- 'return',
- 'switch',
- 'try',
- 'var',
- 'void',
- 'while'
- ]);
-
bool looksLikeStartOfNextTopLevelDeclaration(Token token) =>
token.isTopLevelKeyword ||
isOneOfOrEof(token, const ['const', 'get', 'final', 'set', 'var', 'void']);
diff --git a/pkg/front_end/lib/src/fasta/parser/parser.dart b/pkg/front_end/lib/src/fasta/parser/parser.dart
index 170db1e..6178e4b 100644
--- a/pkg/front_end/lib/src/fasta/parser/parser.dart
+++ b/pkg/front_end/lib/src/fasta/parser/parser.dart
@@ -63,7 +63,8 @@
import 'forwarding_listener.dart' show ForwardingListener;
-import 'identifier_context.dart' show IdentifierContext;
+import 'identifier_context.dart'
+ show IdentifierContext, looksLikeExpressionStart;
import 'listener.dart' show Listener;
@@ -1903,27 +1904,6 @@
return identifier;
}
- /// Return `true` if the given [token] should be treated like the start of
- /// an expression for the purposes of recovery.
- bool isExpressionStartForRecovery(Token next) =>
- next.isKeywordOrIdentifier ||
- next.type == TokenType.DOUBLE ||
- next.type == TokenType.HASH ||
- next.type == TokenType.HEXADECIMAL ||
- next.type == TokenType.IDENTIFIER ||
- next.type == TokenType.INT ||
- next.type == TokenType.STRING ||
- optional('{', next) ||
- optional('(', next) ||
- optional('[', next) ||
- optional('[]', next) ||
- optional('<', next) ||
- optional('!', next) ||
- optional('-', next) ||
- optional('~', next) ||
- optional('++', next) ||
- optional('--', next);
-
Token expect(String string, Token token) {
// TODO(danrubel): update all uses of expect(';'...) to ensureSemicolon
// then add assert(!identical(';', string));
@@ -4010,7 +3990,7 @@
}
// Recovery
- if (!isExpressionStartForRecovery(next)) {
+ if (!looksLikeExpressionStart(next)) {
if (beginToken.endGroup.isSynthetic) {
// The scanner has already reported an error,
// but inserted `]` in the wrong place.
@@ -4070,7 +4050,7 @@
break;
}
// Recovery
- if (isExpressionStartForRecovery(next)) {
+ if (looksLikeExpressionStart(next)) {
// If this looks like the start of an expression,
// then report an error, insert the comma, and continue parsing.
next = rewriteAndRecover(
@@ -4487,7 +4467,7 @@
break;
}
// Recovery
- if (isExpressionStartForRecovery(next)) {
+ if (looksLikeExpressionStart(next)) {
// If this looks like the start of an expression,
// then report an error, insert the comma, and continue parsing.
next = rewriteAndRecover(
