commit 63001295c009865c5f0dffbabe82251280b29143
author Paul Berry <paulberry@google.com> Fri May 10 17:49:57 2019 +0000
committer commit-bot@chromium.org <commit-bot@chromium.org> Fri May 10 17:49:57 2019 +0000
tree ba7418adc6c5f1a5b83a1b48b4d571e1af4b3ce8
parent 7e0b680851a59ff373708ac2c66e62ca9ace9f69

optionally report scanner errors before parsing

The fasta parser handles error tokens produced by the scanner
but the old parser expect error tokens to have been already
translated into error messages. This updates the analyzer scanner
to optionally translate scanner errors.

Change-Id: I62383965285511c93e8fbf6a4cada97e9e4e3c04
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/102143
Commit-Queue: Paul Berry <paulberry@google.com>
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>

pkg/analyzer/lib/src/dart/scanner/scanner.dart

diff --git a/pkg/analyzer/lib/src/dart/scanner/scanner.dart b/pkg/analyzer/lib/src/dart/scanner/scanner.dart
index 64a0b86..0d45a9e 100644
--- a/pkg/analyzer/lib/src/dart/scanner/scanner.dart
+++ b/pkg/analyzer/lib/src/dart/scanner/scanner.dart
@@ -9,7 +9,8 @@
 import 'package:analyzer/src/dart/scanner/reader.dart';
 import 'package:analyzer/src/generated/source.dart';
 import 'package:front_end/src/fasta/scanner.dart' as fasta;
-import 'package:front_end/src/scanner/token.dart' show Token;
+import 'package:front_end/src/scanner/errors.dart' show translateErrorToken;
+import 'package:front_end/src/scanner/token.dart' show Token, TokenType;
 import 'package:pub_semver/pub_semver.dart';
 
 export 'package:analyzer/src/dart/error/syntactic_errors.dart';
@@ -137,7 +138,7 @@
     lineStarts.add(offset - column + 1);
   }
 
-  Token tokenize() {
+  Token tokenize({bool reportScannerErrors = true}) {
     fasta.ScannerResult result = fasta.scanString(_contents,
         configuration: _featureSet != null
             ? buildConfig(_featureSet)
@@ -155,6 +156,19 @@
 
     lineStarts.addAll(result.lineStarts);
     fasta.Token token = result.tokens;
+
+    // The fasta parser handles error tokens produced by the scanner
+    // but the old parser used by angular does not
+    // and expects that scanner errors to be reported here
+    if (reportScannerErrors) {
+      // The default recovery strategy used by scanString
+      // places all error tokens at the head of the stream.
+      while (token.type == TokenType.BAD_INPUT) {
+        translateErrorToken(token, reportError);
+        token = token.next;
+      }
+    }
+
     firstToken = token;
     // Update all token offsets based upon the reader's starting offset
     if (_readerOffset != -1) {

pkg/analyzer/test/generated/scanner_test.dart

diff --git a/pkg/analyzer/test/generated/scanner_test.dart b/pkg/analyzer/test/generated/scanner_test.dart
index d73d00d..5470859 100644
--- a/pkg/analyzer/test/generated/scanner_test.dart
+++ b/pkg/analyzer/test/generated/scanner_test.dart
@@ -147,7 +147,7 @@
     Scanner scanner =
         new Scanner(null, new CharSequenceReader(source), listener)
           ..configureFeatures(featureSet);
-    Token token = scanner.tokenize();
+    Token token = scanner.tokenize(reportScannerErrors: false);
     expect(token, TypeMatcher<UnmatchedToken>());
     token = token.next;
     expect(token, TypeMatcher<UnmatchedToken>());