Add scanner and parser support for new >>> operator
This adds support for the new >>> operator as detailed in
https://github.com/dart-lang/language/blob/master/accepted/future-releases/constant-update-2018/feature-specification.md#new-operators
Tracking issue: https://github.com/dart-lang/language/issues/60
Enable support by setting AbstractScanner.enableGtGtGt = true
Change-Id: I8e7d2fb341971452d8e05d734a1de612880fef5e
Reviewed-on: https://dart-review.googlesource.com/c/85421
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>
diff --git a/pkg/analyzer/test/generated/parser_fasta_test.dart b/pkg/analyzer/test/generated/parser_fasta_test.dart
index 1bcac17..2e121cc 100644
--- a/pkg/analyzer/test/generated/parser_fasta_test.dart
+++ b/pkg/analyzer/test/generated/parser_fasta_test.dart
@@ -48,7 +48,56 @@
@reflectiveTest
class ClassMemberParserTest_Fasta extends FastaParserTestCase
- with ClassMemberParserTestMixin {}
+ with ClassMemberParserTestMixin {
+ void test_parseClassMember_operator_gtgtgt() {
+ final sourceText = 'class C { bool operator >>>(other) => false; }';
+
+ // ---------------------------------------------------
+ // TODO(danrubel): Replace this section with a call to parseCompilationUnit
+ // once '>>>' token support is enabled permanently.
+
+ var source = new StringSource(sourceText, 'parser_test_StringSource.dart');
+ GatheringErrorListener errorListener =
+ new GatheringErrorListener(checkRanges: true);
+
+ // Scan tokens
+ StringScanner scanner = new StringScanner(sourceText, includeComments: true)
+ ..enableGtGtGt = true;
+ Token tokens = scanner.tokenize();
+ expect(scanner.hasErrors, isFalse);
+
+ // Run parser
+ ErrorReporter errorReporter = new ErrorReporter(errorListener, source);
+ fasta.Parser parser = new fasta.Parser(null);
+ AstBuilder astBuilder = new AstBuilder(errorReporter, source.uri, true);
+ parser.listener = astBuilder;
+ astBuilder.parser = parser;
+ parser.parseUnit(tokens);
+
+ CompilationUnitImpl unit = astBuilder.pop();
+ expect(unit, isNotNull);
+ unit.localDeclarations = astBuilder.localDeclarations;
+ errorListener.assertNoErrors();
+
+ // ---------------------------------------------------
+
+ ClassDeclaration declaration = unit.declarations[0];
+ ClassMember member = declaration.members[0];
+ expect(member, isNotNull);
+ expect(member, new TypeMatcher<MethodDeclaration>());
+ MethodDeclaration method = member;
+ expect(method.documentationComment, isNull);
+ expect(method.externalKeyword, isNull);
+ expect(method.modifierKeyword, isNull);
+ expect(method.propertyKeyword, isNull);
+ expect(method.returnType, isNotNull);
+ expect(method.name.name, '>>>');
+ expect(method.operatorKeyword, isNotNull);
+ expect(method.typeParameters, isNull);
+ expect(method.parameters, isNotNull);
+ expect(method.body, isNotNull);
+ }
+}
/**
* Tests of the fasta parser based on [ComplexParserTestMixin].
diff --git a/pkg/front_end/lib/src/fasta/parser/parser.dart b/pkg/front_end/lib/src/fasta/parser/parser.dart
index edfb5c7..bc96678 100644
--- a/pkg/front_end/lib/src/fasta/parser/parser.dart
+++ b/pkg/front_end/lib/src/fasta/parser/parser.dart
@@ -263,7 +263,15 @@
/// Experimental flag for enabling set literal support.
/// See https://github.com/dart-lang/sdk/issues/35121
- bool parseSetLiterals = false;
+ bool enableSetLiterals = false;
+
+ /// Obsolete experimental flag for enabling set literal support.
+ /// Use enableSetLiterals instead.
+ /// TODO(danrubel): Remove this once this has been merged into the analyzer
+ /// branch and the references to this have been cleaned up.
+ set parseSetLiterals(bool value) {
+ enableSetLiterals = value;
+ }
/// Represents parser state: what asynchronous syntax is allowed in the
/// function being currently parsed. In rare situations, this can be set by
@@ -4200,7 +4208,7 @@
/// This method parses the portion of a set or map literal that starts with
/// the left curly brace when there are no leading type arguments.
Token parseLiteralSetOrMapSuffix(final Token start, Token constKeyword) {
- if (!parseSetLiterals) {
+ if (!enableSetLiterals) {
// TODO(danrubel): remove this once set literals are permanent
return parseLiteralMapSuffix(start, constKeyword);
}
@@ -4323,7 +4331,7 @@
/// if not. This is a suffix parser because it is assumed that type arguments
/// have been parsed, or `listener.handleNoTypeArguments` has been executed.
Token parseLiteralSetSuffix(Token token, Token constKeyword) {
- if (!parseSetLiterals) {
+ if (!enableSetLiterals) {
// TODO(danrubel): remove this once set literals are permanent
return parseLiteralMapSuffix(token, constKeyword);
}
diff --git a/pkg/front_end/lib/src/fasta/parser/type_info.dart b/pkg/front_end/lib/src/fasta/parser/type_info.dart
index 9e43321..585fedf3 100644
--- a/pkg/front_end/lib/src/fasta/parser/type_info.dart
+++ b/pkg/front_end/lib/src/fasta/parser/type_info.dart
@@ -26,27 +26,27 @@
/// Call this function when the token after [token] must be a type (not void).
/// This function will call the appropriate event methods on the [Parser]'s
/// listener to handle the type, inserting a synthetic type reference if
- /// necessary. This may modify the token stream when parsing `>>` in valid
- /// code or during recovery.
+ /// necessary. This may modify the token stream when parsing `>>` or `>>>`
+ /// in valid code or during recovery.
Token ensureTypeNotVoid(Token token, Parser parser);
/// Call this function when the token after [token] must be a type or void.
/// This function will call the appropriate event methods on the [Parser]'s
/// listener to handle the type, inserting a synthetic type reference if
- /// necessary. This may modify the token stream when parsing `>>` in valid
- /// code or during recovery.
+ /// necessary. This may modify the token stream when parsing `>>` or `>>>`
+ /// in valid code or during recovery.
Token ensureTypeOrVoid(Token token, Parser parser);
/// Call this function to parse an optional type (not void) after [token].
/// This function will call the appropriate event methods on the [Parser]'s
/// listener to handle the type. This may modify the token stream
- /// when parsing `>>` in valid code or during recovery.
+ /// when parsing `>>` or `>>>` in valid code or during recovery.
Token parseTypeNotVoid(Token token, Parser parser);
/// Call this function to parse an optional type or void after [token].
/// This function will call the appropriate event methods on the [Parser]'s
/// listener to handle the type. This may modify the token stream
- /// when parsing `>>` in valid code or during recovery.
+ /// when parsing `>>` or `>>>` in valid code or during recovery.
Token parseType(Token token, Parser parser);
/// Call this function with the [token] before the type to obtain
@@ -76,14 +76,14 @@
/// Call this function to parse optional type arguments after [token].
/// This function will call the appropriate event methods on the [Parser]'s
/// listener to handle the arguments. This may modify the token stream
- /// when parsing `>>` in valid code or during recovery.
+ /// when parsing `>>` or `>>>` in valid code or during recovery.
Token parseArguments(Token token, Parser parser);
/// Call this function to parse optional type parameters
/// (also known as type variables) after [token].
/// This function will call the appropriate event methods on the [Parser]'s
/// listener to handle the parameters. This may modify the token stream
- /// when parsing `>>` in valid code or during recovery.
+ /// when parsing `>>` or `>>>` in valid code or during recovery.
Token parseVariables(Token token, Parser parser);
/// Call this function with the [token] before the type var to obtain
diff --git a/pkg/front_end/lib/src/fasta/parser/type_info_impl.dart b/pkg/front_end/lib/src/fasta/parser/type_info_impl.dart
index 7f959e9..f46d819 100644
--- a/pkg/front_end/lib/src/fasta/parser/type_info_impl.dart
+++ b/pkg/front_end/lib/src/fasta/parser/type_info_impl.dart
@@ -30,6 +30,7 @@
skipMetadata,
splitGtEq,
splitGtFromGtGtEq,
+ splitGtFromGtGtGt,
splitGtGt,
syntheticGt;
@@ -990,16 +991,17 @@
}
}
-/// Return `true` if [token] is one of `>`, `>>`, `>=', or `>>=`.
+/// Return `true` if [token] is one of `>`, `>>`, `>=`, `>>>`, or `>>=`.
bool isCloser(Token token) {
final value = token.stringValue;
return identical(value, '>') ||
identical(value, '>>') ||
identical(value, '>=') ||
+ identical(value, '>>>') ||
identical(value, '>>=');
}
-/// If [beforeCloser].next is one of `>`, `>>`, `>=', or `>>=`,
+/// If [beforeCloser].next is one of `>`, `>>`, `>=`, `>>>`, or `>>=`,
/// then update the token stream and return `true`.
bool parseCloser(Token beforeCloser) {
Token unsplit = beforeCloser.next;
@@ -1015,7 +1017,7 @@
}
/// If [closer] is `>` then return it.
-/// If [closer] is one of `>>`, `>=', or `>>=` then split then token
+/// If [closer] is one of `>>`, `>=`, `>>>`, or `>>=` then split then token
/// and return the leading `>` without updating the token stream.
/// If [closer] is none of the above, then return null;
Token splitCloser(Token closer) {
@@ -1026,6 +1028,8 @@
return splitGtGt(closer);
} else if (identical(value, '>=')) {
return splitGtEq(closer);
+ } else if (identical(value, '>>>')) {
+ return splitGtFromGtGtGt(closer);
} else if (identical(value, '>>=')) {
return splitGtFromGtGtEq(closer);
}
diff --git a/pkg/front_end/lib/src/fasta/parser/util.dart b/pkg/front_end/lib/src/fasta/parser/util.dart
index 50937cb6..e02385f 100644
--- a/pkg/front_end/lib/src/fasta/parser/util.dart
+++ b/pkg/front_end/lib/src/fasta/parser/util.dart
@@ -178,6 +178,18 @@
..next = token.next);
}
+/// Split `>>>` into two separate tokens... `>` followed by `>>`.
+/// Call [Token.setNext] to add the token to the stream.
+Token splitGtFromGtGtGt(Token token) {
+ assert(optional('>>>', token));
+ return new SimpleToken(
+ TokenType.GT, token.charOffset, token.precedingComments)
+ ..setNext(new SimpleToken(TokenType.GT_GT, token.charOffset + 1)
+ // Set next rather than calling Token.setNext
+ // so that the previous token is not set.
+ ..next = token.next);
+}
+
/// Return a synthetic `<` followed by [next].
/// Call [Token.setNext] to add the token to the stream.
Token syntheticGt(Token next) {
diff --git a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
index 55266e8..6ac7640 100644
--- a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
@@ -44,6 +44,11 @@
final bool includeComments;
+ /// Experimental flag for enabling parsing of `>>>`.
+ /// See https://github.com/dart-lang/language/issues/61
+ /// and https://github.com/dart-lang/language/issues/60
+ bool enableGtGtGt = false;
+
/**
* The string offset for the next token that will be created.
*
@@ -638,7 +643,7 @@
}
int tokenizeGreaterThan(int next) {
- // > >= >> >>=
+ // > >= >> >>= >>>
next = advance();
if (identical($EQ, next)) {
appendPrecedenceToken(TokenType.GT_EQ);
@@ -648,6 +653,9 @@
if (identical($EQ, next)) {
appendPrecedenceToken(TokenType.GT_GT_EQ);
return advance();
+ } else if (enableGtGtGt && identical($GT, next)) {
+ appendPrecedenceToken(TokenType.GT_GT_GT);
+ return advance();
} else {
appendGtGt(TokenType.GT_GT);
return next;
diff --git a/pkg/front_end/test/fasta/parser/type_info_test.dart b/pkg/front_end/test/fasta/parser/type_info_test.dart
index f3f7c2c..975eda8 100644
--- a/pkg/front_end/test/fasta/parser/type_info_test.dart
+++ b/pkg/front_end/test/fasta/parser/type_info_test.dart
@@ -2,11 +2,15 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
+import 'dart:convert';
+
import 'package:front_end/src/fasta/messages.dart';
import 'package:front_end/src/fasta/parser.dart';
import 'package:front_end/src/fasta/parser/type_info.dart';
import 'package:front_end/src/fasta/parser/type_info_impl.dart';
-import 'package:front_end/src/fasta/scanner.dart';
+import 'package:front_end/src/fasta/scanner.dart' hide scanString;
+import 'package:front_end/src/fasta/scanner/recover.dart'
+ show defaultRecoveryStrategy;
import 'package:front_end/src/scanner/token.dart';
import 'package:test/test.dart';
import 'package:test_reflective_loader/test_reflective_loader.dart';
@@ -26,6 +30,35 @@
});
}
+/// TODO(danrubel): Remove this and use scanner.dart scanString
+/// once support for `>>>` is permanently enabled.
+///
+/// Scan/tokenize the given [source].
+/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
+ScannerResult scanString(String source,
+ {bool includeComments: false,
+ bool scanLazyAssignmentOperators: false,
+ Recover recover}) {
+ assert(source != null, 'source must not be null');
+ StringScanner scanner =
+ new StringScanner(source, includeComments: includeComments)
+ ..enableGtGtGt = true;
+ return _tokenizeAndRecover(scanner, recover, source: source);
+}
+
+/// TODO(danrubel): Remove this once support for `>>>` is permanently enabled.
+ScannerResult _tokenizeAndRecover(Scanner scanner, Recover recover,
+ {List<int> bytes, String source}) {
+ Token tokens = scanner.tokenize();
+ if (scanner.hasErrors) {
+ if (bytes == null) bytes = utf8.encode(source);
+ recover ??= defaultRecoveryStrategy;
+ tokens = recover(bytes, tokens, scanner.lineStarts);
+ }
+ return new ScannerResult(
+ tokens, scanner.lineStarts, scanner.hasErrors, scanner.errors);
+}
+
@reflectiveTest
class NoTypeInfoTest {
void test_basic() {
@@ -1250,6 +1283,39 @@
'handleType S',
'endTypeArguments 1 < >'
]);
+ expectComplexTypeArg('<S<T<U>>>', typeArgumentCount: 1, expectedCalls: [
+ 'beginTypeArguments <',
+ 'handleIdentifier S typeReference',
+ 'beginTypeArguments <',
+ 'handleIdentifier T typeReference',
+ 'beginTypeArguments <',
+ 'handleIdentifier U typeReference',
+ 'handleNoTypeArguments >>>',
+ 'handleType U',
+ 'endTypeArguments 1 < >',
+ 'handleType T',
+ 'endTypeArguments 1 < >',
+ 'handleType S',
+ 'endTypeArguments 1 < >'
+ ]);
+ expectComplexTypeArg('<S<T<U,V>>>', typeArgumentCount: 1, expectedCalls: [
+ 'beginTypeArguments <',
+ 'handleIdentifier S typeReference',
+ 'beginTypeArguments <',
+ 'handleIdentifier T typeReference',
+ 'beginTypeArguments <',
+ 'handleIdentifier U typeReference',
+ 'handleNoTypeArguments ,',
+ 'handleType U',
+ 'handleIdentifier V typeReference',
+ 'handleNoTypeArguments >>>',
+ 'handleType V',
+ 'endTypeArguments 2 < >',
+ 'handleType T',
+ 'endTypeArguments 1 < >',
+ 'handleType S',
+ 'endTypeArguments 1 < >'
+ ]);
expectComplexTypeArg('<S<Function()>>',
typeArgumentCount: 1,
expectedCalls: [
@@ -1299,6 +1365,26 @@
'handleType S',
'endTypeArguments 1 < >'
]);
+ expectComplexTypeArg('<S<T<void Function()>>>',
+ typeArgumentCount: 1,
+ expectedCalls: [
+ 'beginTypeArguments <',
+ 'handleIdentifier S typeReference',
+ 'beginTypeArguments <',
+ 'handleIdentifier T typeReference',
+ 'beginTypeArguments <',
+ 'handleNoTypeVariables (',
+ 'beginFunctionType void', // was 'beginFunctionType Function'
+ 'handleVoidKeyword void', // was 'handleNoType <'
+ 'beginFormalParameters ( MemberKind.GeneralizedFunctionType',
+ 'endFormalParameters 0 ( ) MemberKind.GeneralizedFunctionType',
+ 'endFunctionType Function',
+ 'endTypeArguments 1 < >',
+ 'handleType T',
+ 'endTypeArguments 1 < >',
+ 'handleType S',
+ 'endTypeArguments 1 < >'
+ ]);
}
void test_computeTypeArg_complex_recovery() {
@@ -1751,7 +1837,7 @@
'handleIdentifier List typeReference',
'beginTypeArguments <',
'handleIdentifier T typeReference',
- 'handleNoTypeArguments >',
+ 'handleNoTypeArguments >>>',
'handleType T',
'endTypeArguments 1 < >',
'handleType List',
@@ -1760,6 +1846,32 @@
'endTypeVariable > 0 extends',
'endTypeVariables < >'
]);
+ expectComplexTypeParam('<T extends List<Map<S, T>>>',
+ typeArgumentCount: 1,
+ expectedCalls: [
+ 'beginTypeVariables <',
+ 'beginMetadataStar T',
+ 'endMetadataStar 0',
+ 'handleIdentifier T typeVariableDeclaration',
+ 'beginTypeVariable T',
+ 'handleTypeVariablesDefined > 1',
+ 'handleIdentifier List typeReference',
+ 'beginTypeArguments <',
+ 'handleIdentifier Map typeReference',
+ 'beginTypeArguments <',
+ 'handleIdentifier S typeReference',
+ 'handleNoTypeArguments ,',
+ 'handleType S',
+ 'handleIdentifier T typeReference',
+ 'handleNoTypeArguments >>>',
+ 'handleType T',
+ 'endTypeArguments 2 < >',
+ 'handleType Map',
+ 'endTypeArguments 1 < >',
+ 'handleType List',
+ 'endTypeVariable > 0 extends',
+ 'endTypeVariables < >'
+ ]);
}
void test_computeTypeParam_34850() {
