Add scanner languageVersionChanged callback
Change-Id: I64835e303efd83ac1f7cb0015407be8c196a5249
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/100925
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>
diff --git a/pkg/front_end/lib/src/fasta/scanner.dart b/pkg/front_end/lib/src/fasta/scanner.dart
index a6e262a..f9f1741 100644
--- a/pkg/front_end/lib/src/fasta/scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner.dart
@@ -8,7 +8,8 @@
import '../scanner/token.dart' show Token;
-import 'scanner/abstract_scanner.dart' show ScannerConfiguration;
+import 'scanner/abstract_scanner.dart'
+ show LanguageVersionChanged, ScannerConfiguration;
import 'scanner/string_scanner.dart' show StringScanner;
@@ -18,7 +19,8 @@
import 'scanner/recover.dart' show defaultRecoveryStrategy;
-export 'scanner/abstract_scanner.dart' show ScannerConfiguration;
+export 'scanner/abstract_scanner.dart'
+ show LanguageVersionChanged, ScannerConfiguration;
export 'scanner/token.dart'
show
@@ -33,6 +35,8 @@
export 'scanner/error_token.dart'
show ErrorToken, buildUnexpectedCharacterToken;
+export 'scanner/token.dart' show LanguageVersionToken;
+
export 'scanner/token_constants.dart' show EOF_TOKEN;
export 'scanner/utf8_bytes_scanner.dart' show Utf8BytesScanner;
@@ -55,6 +59,9 @@
List<int> get lineStarts;
+ /// Configure which tokens are produced.
+ set configuration(ScannerConfiguration config);
+
Token tokenize();
}
@@ -71,14 +78,17 @@
/// Scan/tokenize the given UTF8 [bytes].
/// If [recover] is null, then the [defaultRecoveryStrategy] is used.
ScannerResult scan(List<int> bytes,
- {bool includeComments: false,
- ScannerConfiguration configuration,
+ {ScannerConfiguration configuration,
+ bool includeComments: false,
+ LanguageVersionChanged languageVersionChanged,
Recover recover}) {
if (bytes.last != 0) {
throw new ArgumentError("[bytes]: the last byte must be null.");
}
Scanner scanner = new Utf8BytesScanner(bytes,
- configuration: configuration, includeComments: includeComments);
+ configuration: configuration,
+ includeComments: includeComments,
+ languageVersionChanged: languageVersionChanged);
return _tokenizeAndRecover(scanner, recover, bytes: bytes);
}
@@ -87,11 +97,14 @@
ScannerResult scanString(String source,
{ScannerConfiguration configuration,
bool includeComments: false,
+ LanguageVersionChanged languageVersionChanged,
bool scanLazyAssignmentOperators: false,
Recover recover}) {
assert(source != null, 'source must not be null');
StringScanner scanner = new StringScanner(source,
- configuration: configuration, includeComments: includeComments);
+ configuration: configuration,
+ includeComments: includeComments,
+ languageVersionChanged: languageVersionChanged);
return _tokenizeAndRecover(scanner, recover, source: source);
}
diff --git a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
index fd29cd4..79ba63d 100644
--- a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
@@ -32,6 +32,9 @@
import 'characters.dart';
+typedef void LanguageVersionChanged(
+ Scanner scanner, LanguageVersionToken languageVersion);
+
abstract class AbstractScanner implements Scanner {
/**
* A flag indicating whether character sequences `&&=` and `||=`
@@ -43,6 +46,11 @@
final bool includeComments;
+ /// Called when the scanner detects a language version comment
+ /// so that the listener can update the scanner configuration
+ /// based upon the specified language version.
+ final LanguageVersionChanged languageVersionChanged;
+
/// Experimental flag for enabling scanning of `>>>`.
/// See https://github.com/dart-lang/language/issues/61
/// and https://github.com/dart-lang/language/issues/60
@@ -98,15 +106,14 @@
final List<int> lineStarts;
AbstractScanner(ScannerConfiguration config, this.includeComments,
+ this.languageVersionChanged,
{int numberOfBytesHint})
: lineStarts = new LineStarts(numberOfBytesHint) {
this.tail = this.tokens;
this.configuration = config;
}
- /**
- * Configure which tokens are produced.
- */
+ @override
set configuration(ScannerConfiguration config) {
if (config != null) {
_enableNonNullable = config.enableNonNullable;
@@ -320,7 +327,7 @@
}
int bigHeaderSwitch(int next) {
- if (languageVersion != null || !identical(next, $SLASH)) {
+ if (!identical(next, $SLASH)) {
return bigSwitch(next);
}
beginToken();
@@ -970,7 +977,14 @@
}
languageVersion = createLanguageVersionToken(start, major, minor);
- configuration = ScannerConfiguration.from(languageVersion);
+ if (languageVersionChanged != null) {
+ // TODO(danrubel): make this required and remove the languageVersion field
+ languageVersionChanged(this, languageVersion);
+ } else {
+ // TODO(danrubel): remove this hack and require listener to update
+ // the scanner's configuration.
+ configuration = ScannerConfiguration.classic;
+ }
if (includeComments) {
_appendToCommentStream(languageVersion);
}
@@ -1528,12 +1542,6 @@
static const classic = ScannerConfiguration();
static const nonNullable = ScannerConfiguration(enableNonNullable: true);
- /// Return the scanner configuration for the given language version.
- static ScannerConfiguration from(LanguageVersionToken languageVersion) {
- // TODO(danrubel): update this to return config for new releases
- return classic;
- }
-
/// Experimental flag for enabling scanning of NNBD tokens
/// such as 'required' and 'late'
final bool enableNonNullable;
diff --git a/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
index 66e7e39..22df483 100644
--- a/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
@@ -21,7 +21,11 @@
import 'characters.dart' show $LF, $STX;
import 'abstract_scanner.dart'
- show AbstractScanner, ScannerConfiguration, closeBraceInfoFor;
+ show
+ AbstractScanner,
+ LanguageVersionChanged,
+ ScannerConfiguration,
+ closeBraceInfoFor;
import '../util/link.dart' show Link;
@@ -29,8 +33,9 @@
bool hasErrors = false;
ArrayBasedScanner(ScannerConfiguration config, bool includeComments,
- {int numberOfBytesHint})
- : super(config, includeComments, numberOfBytesHint: numberOfBytesHint);
+ LanguageVersionChanged languageVersionChanged, {int numberOfBytesHint})
+ : super(config, includeComments, languageVersionChanged,
+ numberOfBytesHint: numberOfBytesHint);
/**
* The stack of open groups, e.g [: { ... ( .. :]
diff --git a/pkg/front_end/lib/src/fasta/scanner/string_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/string_scanner.dart
index 2b5c472..7fa5203 100644
--- a/pkg/front_end/lib/src/fasta/scanner/string_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/string_scanner.dart
@@ -8,7 +8,8 @@
import '../../scanner/token.dart' as analyzer show StringToken;
-import 'abstract_scanner.dart' show ScannerConfiguration;
+import 'abstract_scanner.dart'
+ show LanguageVersionChanged, ScannerConfiguration;
import 'array_based_scanner.dart' show ArrayBasedScanner;
@@ -29,9 +30,11 @@
int scanOffset = -1;
StringScanner(String string,
- {ScannerConfiguration configuration, bool includeComments: false})
+ {ScannerConfiguration configuration,
+ bool includeComments: false,
+ LanguageVersionChanged languageVersionChanged})
: string = ensureZeroTermination(string),
- super(configuration, includeComments);
+ super(configuration, includeComments, languageVersionChanged);
static String ensureZeroTermination(String string) {
return (string.isEmpty || string.codeUnitAt(string.length - 1) != 0)
diff --git a/pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart
index 9aabe30..e847390 100644
--- a/pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/utf8_bytes_scanner.dart
@@ -12,7 +12,8 @@
import '../scanner.dart' show unicodeReplacementCharacter;
-import 'abstract_scanner.dart' show ScannerConfiguration;
+import 'abstract_scanner.dart'
+ show LanguageVersionChanged, ScannerConfiguration;
import 'token.dart'
show CommentToken, DartDocToken, LanguageVersionToken, StringToken;
@@ -85,8 +86,11 @@
* is not the case, the entire array is copied before scanning.
*/
Utf8BytesScanner(this.bytes,
- {ScannerConfiguration configuration, bool includeComments: false})
- : super(configuration, includeComments, numberOfBytesHint: bytes.length) {
+ {ScannerConfiguration configuration,
+ bool includeComments: false,
+ LanguageVersionChanged languageVersionChanged})
+ : super(configuration, includeComments, languageVersionChanged,
+ numberOfBytesHint: bytes.length) {
assert(bytes.last == 0);
// Skip a leading BOM.
if (containsBomAt(0)) byteOffset += 3;
diff --git a/pkg/front_end/test/scanner_fasta_test.dart b/pkg/front_end/test/scanner_fasta_test.dart
index d4f785b..914ae8c 100644
--- a/pkg/front_end/test/scanner_fasta_test.dart
+++ b/pkg/front_end/test/scanner_fasta_test.dart
@@ -717,15 +717,26 @@
ScannerResult scanSource(source, {includeComments: true}) {
List<int> encoded = utf8.encode(source).toList(growable: true);
encoded.add(0); // Ensure 0 terminted bytes for UTF8 scanner
- return usedForFuzzTesting.scan(encoded, includeComments: includeComments);
+ return usedForFuzzTesting.scan(encoded,
+ includeComments: includeComments,
+ languageVersionChanged: languageVersionChanged);
}
}
/// Scanner tests that exercise the Fasta scanner directly.
@reflectiveTest
class ScannerTest_Fasta_Direct extends ScannerTest_Fasta_Base {
+ LanguageVersionToken languageVersion;
+
+ void languageVersionChanged(
+ Scanner scanner, LanguageVersionToken languageVersion) {
+ this.languageVersion = languageVersion;
+ }
+
ScannerResult scanSource(source, {includeComments: true}) =>
- scanString(source, includeComments: includeComments);
+ scanString(source,
+ includeComments: includeComments,
+ languageVersionChanged: languageVersionChanged);
@override
Token scan(String source) {
