encapsulate scanner configuration in ScannerConfiguration
... and update TypeInfoTest to use ScannerConfiguration
Change-Id: I637d6fbeecd0b6d16c37e57cfd8ba9fc1fba494a
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/100791
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>
diff --git a/pkg/front_end/lib/src/fasta/scanner.dart b/pkg/front_end/lib/src/fasta/scanner.dart
index e3947f1..ae96681 100644
--- a/pkg/front_end/lib/src/fasta/scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner.dart
@@ -8,12 +8,16 @@
import '../scanner/token.dart' show Token;
+import 'scanner/abstract_scanner.dart' show ScannerConfiguration;
+
import 'scanner/string_scanner.dart' show StringScanner;
import 'scanner/utf8_bytes_scanner.dart' show Utf8BytesScanner;
import 'scanner/recover.dart' show defaultRecoveryStrategy;
+export 'scanner/abstract_scanner.dart' show ScannerConfiguration;
+
export 'scanner/token.dart'
show
StringToken,
@@ -58,32 +62,42 @@
/// Scan/tokenize the given UTF8 [bytes].
/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
ScannerResult scan(List<int> bytes,
- {bool includeComments: false, Recover recover}) {
+ {bool includeComments: false,
+ ScannerConfiguration configuration,
+ Recover recover}) {
if (bytes.last != 0) {
throw new ArgumentError("[bytes]: the last byte must be null.");
}
- Scanner scanner =
- new Utf8BytesScanner(bytes, includeComments: includeComments);
+ Scanner scanner = new Utf8BytesScanner(bytes,
+ configuration: configuration, includeComments: includeComments);
return _tokenizeAndRecover(scanner, recover, bytes: bytes);
}
/// Scan/tokenize the given [source].
/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
ScannerResult scanString(String source,
- {bool enableGtGtGt: false,
- bool enableGtGtGtEq: false,
- bool enableNonNullable: false,
+ {bool enableGtGtGt,
+ bool enableGtGtGtEq,
+ bool enableNonNullable,
+ ScannerConfiguration configuration,
bool includeComments: false,
bool scanLazyAssignmentOperators: false,
Recover recover}) {
// TODO(brianwilkerson): Remove the parameter `enableGtGtGt` after the feature
// has been anabled by default.
assert(source != null, 'source must not be null');
- StringScanner scanner =
- new StringScanner(source, includeComments: includeComments);
- scanner.enableGtGtGt = enableGtGtGt;
- scanner.enableGtGtGtEq = enableGtGtGtEq;
- scanner.enableNonNullable = enableNonNullable;
+ StringScanner scanner = new StringScanner(source,
+ configuration: configuration, includeComments: includeComments);
+ // TODO(danrubel): remove these flags and use configuration instead.
+ if (enableGtGtGt != null) {
+ scanner.enableGtGtGt = enableGtGtGt;
+ }
+ if (enableGtGtGtEq != null) {
+ scanner.enableGtGtGtEq = enableGtGtGtEq;
+ }
+ if (enableNonNullable != null) {
+ scanner.enableNonNullable = enableNonNullable;
+ }
return _tokenizeAndRecover(scanner, recover, source: source);
}
diff --git a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
index 65f9f6e..19f1dee 100644
--- a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
@@ -94,9 +94,22 @@
final List<int> lineStarts;
- AbstractScanner(this.includeComments, {int numberOfBytesHint})
+ AbstractScanner(ScannerConfiguration config, this.includeComments,
+ {int numberOfBytesHint})
: lineStarts = new LineStarts(numberOfBytesHint) {
this.tail = this.tokens;
+ this.configuration = config;
+ }
+
+ /**
+ * Configure which tokens are produced based upon the specified configuration.
+ */
+ set configuration(ScannerConfiguration config) {
+ if (config != null) {
+ enableNonNullable = config.enableNonNullable;
+ enableGtGtGt = config.enableGtGtGt;
+ enableGtGtGtEq = config.enableGtGtGtEq;
+ }
}
/**
@@ -1378,3 +1391,31 @@
array = newArray;
}
}
+
+/// [ScannerConfiguration] contains information for configuring which tokens
+/// the scanner produces based upon the Dart language level.
+class ScannerConfiguration {
+ static const classic = ScannerConfiguration();
+
+ /// Experimental flag for enabling scanning of NNBD tokens
+ /// such as 'required' and 'late'
+ final bool enableNonNullable;
+
+ /// Experimental flag for enabling scanning of `>>>`.
+ /// See https://github.com/dart-lang/language/issues/61
+ /// and https://github.com/dart-lang/language/issues/60
+ final bool enableGtGtGt;
+
+ /// Experimental flag for enabling scanning of `>>>=`.
+ /// See https://github.com/dart-lang/language/issues/61
+ /// and https://github.com/dart-lang/language/issues/60
+ final bool enableGtGtGtEq;
+
+ const ScannerConfiguration({
+ bool enableGtGtGt,
+ bool enableGtGtGtEq,
+ bool enableNonNullable,
+ }) : this.enableGtGtGt = enableGtGtGt ?? false,
+ this.enableGtGtGtEq = enableGtGtGtEq ?? false,
+ this.enableNonNullable = enableNonNullable ?? false;
+}
diff --git a/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
index b6e0f09..66e7e39 100644
--- a/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
@@ -20,15 +20,17 @@
import 'characters.dart' show $LF, $STX;
-import 'abstract_scanner.dart' show AbstractScanner, closeBraceInfoFor;
+import 'abstract_scanner.dart'
+ show AbstractScanner, ScannerConfiguration, closeBraceInfoFor;
import '../util/link.dart' show Link;
abstract class ArrayBasedScanner extends AbstractScanner {
bool hasErrors = false;
- ArrayBasedScanner(bool includeComments, {int numberOfBytesHint})
- : super(includeComments, numberOfBytesHint: numberOfBytesHint);
+ ArrayBasedScanner(ScannerConfiguration config, bool includeComments,
+ {int numberOfBytesHint})
+ : super(config, includeComments, numberOfBytesHint: numberOfBytesHint);
/**
* The stack of open groups, e.g [: { ... ( .. :]
diff --git a/pkg/front_end/lib/src/fasta/scanner/string_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/string_scanner.dart
index 952099d..93fe008 100644
--- a/pkg/front_end/lib/src/fasta/scanner/string_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/string_scanner.dart
@@ -8,6 +8,8 @@
import '../../scanner/token.dart' as analyzer show StringToken;
+import 'abstract_scanner.dart' show ScannerConfiguration;
+
import 'array_based_scanner.dart' show ArrayBasedScanner;
import 'token.dart' show CommentToken, DartDocToken, StringToken;
@@ -25,9 +27,10 @@
/** The current offset in [string]. */
int scanOffset = -1;
- StringScanner(String string, {bool includeComments: false})
+ StringScanner(String string,
+ {ScannerConfiguration configuration, bool includeComments: false})
: string = ensureZeroTermination(string),
- super(includeComments);
+ super(configuration, includeComments);
static String ensureZeroTermination(String string) {
return (string.isEmpty || string.codeUnitAt(string.length - 1) != 0)
diff --git a/pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart
index eec14d4..70f61c8 100644
--- a/pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart
@@ -12,6 +12,8 @@
import '../scanner.dart' show unicodeReplacementCharacter;
+import 'abstract_scanner.dart' show ScannerConfiguration;
+
import 'token.dart' show CommentToken, DartDocToken, StringToken;
import 'array_based_scanner.dart' show ArrayBasedScanner;
@@ -81,8 +83,9 @@
* array whose last element is '0' to signal the end of the file. If this
* is not the case, the entire array is copied before scanning.
*/
- Utf8BytesScanner(this.bytes, {bool includeComments: false})
- : super(includeComments, numberOfBytesHint: bytes.length) {
+ Utf8BytesScanner(this.bytes,
+ {ScannerConfiguration configuration, bool includeComments: false})
+ : super(configuration, includeComments, numberOfBytesHint: bytes.length) {
assert(bytes.last == 0);
// Skip a leading BOM.
if (containsBomAt(0)) byteOffset += 3;
diff --git a/pkg/front_end/test/fasta/parser/type_info_test.dart b/pkg/front_end/test/fasta/parser/type_info_test.dart
index d095d9a..6adff9e 100644
--- a/pkg/front_end/test/fasta/parser/type_info_test.dart
+++ b/pkg/front_end/test/fasta/parser/type_info_test.dart
@@ -2,15 +2,12 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
-import 'dart:convert';
-
import 'package:front_end/src/fasta/messages.dart';
import 'package:front_end/src/fasta/parser.dart';
import 'package:front_end/src/fasta/parser/type_info.dart';
import 'package:front_end/src/fasta/parser/type_info_impl.dart';
import 'package:front_end/src/fasta/scanner.dart' hide scanString;
-import 'package:front_end/src/fasta/scanner/recover.dart'
- show defaultRecoveryStrategy;
+import 'package:front_end/src/fasta/scanner.dart' as scanner;
import 'package:front_end/src/scanner/token.dart';
import 'package:test/test.dart';
import 'package:test_reflective_loader/test_reflective_loader.dart';
@@ -33,34 +30,13 @@
});
}
-/// TODO(danrubel): Remove this and use scanner.dart scanString
-/// once support for `>>>` is permanently enabled.
-///
-/// Scan/tokenize the given [source].
-/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
ScannerResult scanString(String source,
- {bool includeComments: false,
- bool scanLazyAssignmentOperators: false,
- Recover recover}) {
- assert(source != null, 'source must not be null');
- StringScanner scanner =
- new StringScanner(source, includeComments: includeComments)
- ..enableGtGtGt = true
- ..enableGtGtGtEq = true;
- return _tokenizeAndRecover(scanner, recover, source: source);
-}
-
-/// TODO(danrubel): Remove this once support for `>>>` is permanently enabled.
-ScannerResult _tokenizeAndRecover(Scanner scanner, Recover recover,
- {List<int> bytes, String source}) {
- Token tokens = scanner.tokenize();
- if (scanner.hasErrors) {
- if (bytes == null) bytes = utf8.encode(source);
- recover ??= defaultRecoveryStrategy;
- tokens = recover(bytes, tokens, scanner.lineStarts);
- }
- return new ScannerResult(tokens, scanner.lineStarts, scanner.hasErrors);
-}
+ {bool includeComments: false, Recover recover}) =>
+ scanner.scanString(source,
+ configuration: const ScannerConfiguration(
+ enableGtGtGt: true, enableGtGtGtEq: true),
+ includeComments: includeComments,
+ recover: recover);
@reflectiveTest
class NoTypeInfoTest {
