Optionally report scanner errors via errors list rather than in the token stream

This is the first in a multi step process of moving scanner errors
out of the token stream and into a separate errors list produced by the scanner.
As of this CL, if the scanner.reportErrors flag is set then the scanner will report
the unterminated string error via the errors list rather than the token stream.
Once all scanner clients have been updated to process errors reported
through the scanner's error list, then the scanner.reportErrors flag will be removed.

Change-Id: I6ef73df786d98a387cbf2044c4c8325412573f11
Reviewed-on: https://dart-review.googlesource.com/c/80509
Reviewed-by: Peter von der Ahé <ahe@google.com>
Commit-Queue: Dan Rubel <danrubel@google.com>
diff --git a/pkg/front_end/lib/src/fasta/scanner.dart b/pkg/front_end/lib/src/fasta/scanner.dart
index 3229f4a..8e24f03 100644
--- a/pkg/front_end/lib/src/fasta/scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner.dart
@@ -6,6 +6,8 @@
 
 import 'dart:convert' show unicodeReplacementCharacterRune, utf8;
 
+import 'fasta_codes.dart' show LocatedMessage;
+
 import '../scanner/token.dart' show Token;
 
 import 'scanner/string_scanner.dart' show StringScanner;
@@ -39,6 +41,14 @@
 typedef Token Recover(List<int> bytes, Token tokens, List<int> lineStarts);
 
 abstract class Scanner {
+  /// A list of errors that occured during [tokenize] or `null` if none.
+  List<LocatedMessage> errors;
+
+  /// Set true if errors should be reported via the [errors] list.
+  // TODO(danrubel): Remove this once all scanner clients can process
+  // errors reported via the [errors] list.
+  bool reportErrors;
+
   /// Returns true if an error occured during [tokenize].
   bool get hasErrors;
 
@@ -52,7 +62,10 @@
   final List<int> lineStarts;
   final bool hasErrors;
 
-  ScannerResult(this.tokens, this.lineStarts, this.hasErrors);
+  /// Returns a list of errors that occured during [tokenize] or `null` if none.
+  final List<LocatedMessage> errors;
+
+  ScannerResult(this.tokens, this.lineStarts, this.hasErrors, this.errors);
 }
 
 /// Scan/tokenize the given UTF8 [bytes].
@@ -87,5 +100,6 @@
     recover ??= defaultRecoveryStrategy;
     tokens = recover(bytes, tokens, scanner.lineStarts);
   }
-  return new ScannerResult(tokens, scanner.lineStarts, scanner.hasErrors);
+  return new ScannerResult(
+      tokens, scanner.lineStarts, scanner.hasErrors, scanner.errors);
 }
diff --git a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
index 91812bb..55266e8 100644
--- a/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/abstract_scanner.dart
@@ -16,7 +16,8 @@
         messageExpectedHexDigit,
         messageMissingExponent,
         messageUnexpectedDollarInString,
-        messageUnterminatedComment;
+        messageUnterminatedComment,
+        templateUnterminatedString;
 
 import '../scanner.dart'
     show ErrorToken, Keyword, Scanner, buildUnexpectedCharacterToken;
@@ -1241,9 +1242,17 @@
     appendSyntheticSubstringToken(TokenType.STRING, start, asciiOnly, suffix);
     // Ensure that the error is reported on a visible token
     int errorStart = tokenStart < stringOffset ? tokenStart : quoteStart;
-    appendErrorToken(new UnterminatedString(prefix, errorStart, stringOffset));
+    if (reportErrors) {
+      addError(errorStart, stringOffset - errorStart,
+          templateUnterminatedString.withArguments(prefix, suffix));
+    } else {
+      appendErrorToken(
+          new UnterminatedString(prefix, errorStart, stringOffset));
+    }
   }
 
+  void addError(int charOffset, int length, Message message);
+
   int advanceAfterError(bool shouldAdvance) {
     if (atEndOfFile()) return $EOF;
     if (shouldAdvance) {
diff --git a/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart b/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
index b6e0f09..39fba7f 100644
--- a/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/array_based_scanner.dart
@@ -6,6 +6,8 @@
 
 import 'error_token.dart' show ErrorToken, UnmatchedToken;
 
+import '../fasta_codes.dart' show LocatedMessage, Message;
+
 import '../../scanner/token.dart'
     show BeginToken, Keyword, KeywordToken, SyntheticToken, Token, TokenType;
 
@@ -25,6 +27,8 @@
 import '../util/link.dart' show Link;
 
 abstract class ArrayBasedScanner extends AbstractScanner {
+  List<LocatedMessage> errors;
+  bool reportErrors = false;
   bool hasErrors = false;
 
   ArrayBasedScanner(bool includeComments, {int numberOfBytesHint})
@@ -363,4 +367,10 @@
     begin.endGroup = tail;
     appendErrorToken(new UnmatchedToken(begin));
   }
+
+  void addError(int charOffset, int length, Message message) {
+    hasErrors = true;
+    (errors ??= <LocatedMessage>[])
+        .add(new LocatedMessage(null, charOffset, length, message));
+  }
 }
diff --git a/pkg/front_end/lib/src/fasta/scanner/recover.dart b/pkg/front_end/lib/src/fasta/scanner/recover.dart
index d969362..588e658 100644
--- a/pkg/front_end/lib/src/fasta/scanner/recover.dart
+++ b/pkg/front_end/lib/src/fasta/scanner/recover.dart
@@ -209,6 +209,10 @@
     goodTail = current;
   }
 
+  if (error == null) {
+    // All of the errors are in the scanner's error list.
+    return tokens;
+  }
   new Token.eof(-1).setNext(error);
   Token tail;
   if (good != null) {
diff --git a/pkg/front_end/lib/src/scanner/errors.dart b/pkg/front_end/lib/src/scanner/errors.dart
index dc45f52..9e54680 100644
--- a/pkg/front_end/lib/src/scanner/errors.dart
+++ b/pkg/front_end/lib/src/scanner/errors.dart
@@ -118,6 +118,8 @@
     case "UNTERMINATED_STRING_LITERAL":
       // TODO(paulberry,ahe): Fasta reports the error location as the entire
       // string; analyzer expects the end of the string.
+      // TODO(danrubel): Remove this once all analyzer clients
+      // can process errors via the scanner's errors list.
       reportError(
           ScannerErrorCode.UNTERMINATED_STRING_LITERAL, endOffset - 1, null);
       return;
@@ -172,6 +174,22 @@
   }
 }
 
+void translateScanError(
+    Code errorCode, int charOffset, int length, ReportError reportError) {
+  switch (errorCode.analyzerCodes?.first) {
+    case "UNTERMINATED_STRING_LITERAL":
+      // TODO(paulberry,ahe): Fasta reports the error location as the entire
+      // string; analyzer expects the end of the string.
+      reportError(ScannerErrorCode.UNTERMINATED_STRING_LITERAL,
+          charOffset + length - 1, null);
+      break;
+
+    default:
+      throw new UnimplementedError(
+          '$errorCode "${errorCode.analyzerCodes?.first}"');
+  }
+}
+
 /**
  * Determines whether the given [charOffset], which came from the non-EOF token
  * [token], represents the end of the input.
diff --git a/pkg/front_end/test/scanner_fasta_test.dart b/pkg/front_end/test/scanner_fasta_test.dart
index fa5c628..4b68911 100644
--- a/pkg/front_end/test/scanner_fasta_test.dart
+++ b/pkg/front_end/test/scanner_fasta_test.dart
@@ -6,6 +6,7 @@
 import 'package:analyzer/src/fasta/token_utils.dart';
 import 'package:front_end/src/fasta/fasta_codes.dart';
 import 'package:front_end/src/fasta/scanner.dart' as usedForFuzzTesting;
+import 'package:front_end/src/fasta/scanner.dart';
 import 'package:front_end/src/fasta/scanner/error_token.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/string_scanner.dart' as fasta;
 import 'package:front_end/src/fasta/scanner/token.dart' as fasta;
@@ -103,6 +104,14 @@
       {bool lazyAssignmentOperators: false}) {
     var scanner = createScanner(source);
     var token = scanner.tokenize();
+    if (scanner.errors != null) {
+      for (LocatedMessage error in scanner.errors) {
+        translateScanError(error.code, error.charOffset, error.length,
+            (ScannerErrorCode errorCode, int offset, List<Object> arguments) {
+          listener.errors.add(new TestError(offset, errorCode, arguments));
+        });
+      }
+    }
     return new ToAnalyzerTokenStreamConverter_WithListener(listener)
         .convertTokens(token);
   }
@@ -338,7 +347,28 @@
 
 /// Base class for scanner tests that examine the token stream in Fasta format.
 abstract class ScannerTest_Fasta_Base {
-  Token scan(String source);
+  List<LocatedMessage> scanErrors;
+
+  Token scan(String source, {int errorCount});
+
+  expectError(Code code, int charOffset, int length) {
+    if (scanErrors == null) {
+      fail('Expected $code but found no errors');
+    }
+    for (LocatedMessage e in scanErrors) {
+      if (e.code == code && e.charOffset == charOffset && e.length == length) {
+        return;
+      }
+    }
+    final msg = new StringBuffer();
+    msg.writeln('Expected:');
+    msg.writeln('  $code at $charOffset, $length');
+    msg.writeln('but found:');
+    for (LocatedMessage e in scanErrors) {
+      msg.writeln('  ${e.code} at ${e.charOffset}, ${e.length}');
+    }
+    fail(msg.toString());
+  }
 
   expectToken(Token token, TokenType type, int offset, int length,
       {bool isSynthetic: false, String lexeme}) {
@@ -381,7 +411,8 @@
   }
 
   void test_string_simple_unterminated_interpolation_block() {
-    Token token = scan(r'"foo ${bar');
+    Token token = scan(r'"foo ${bar', errorCount: 1);
+    expectError(codeUnterminatedString, 0, 10);
     expectToken(token, TokenType.STRING, 0, 5, lexeme: '"foo ');
 
     token = token.next;
@@ -405,12 +436,12 @@
     expectToken(token, TokenType.STRING, 10, 0, isSynthetic: true, lexeme: '"');
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, '"');
+    expect(token.isEof, isTrue);
   }
 
   void test_string_simple_unterminated_interpolation_block2() {
-    Token token = scan(r'"foo ${bar(baz[');
+    Token token = scan(r'"foo ${bar(baz[', errorCount: 1);
+    expectError(codeUnterminatedString, 0, 15);
     expectToken(token, TokenType.STRING, 0, 5, lexeme: '"foo ');
 
     token = token.next;
@@ -462,12 +493,12 @@
     expectToken(token, TokenType.STRING, 15, 0, isSynthetic: true, lexeme: '"');
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, '"');
+    expect(token.isEof, isTrue);
   }
 
   void test_string_simple_missing_interpolation_identifier() {
-    Token token = scan(r'"foo $');
+    Token token = scan(r'"foo $', errorCount: 1);
+    expectError(codeUnterminatedString, 0, 6);
     expectToken(token, TokenType.STRING, 0, 5, lexeme: '"foo ');
 
     token = token.next;
@@ -485,68 +516,68 @@
     expectToken(token, TokenType.STRING, 6, 0, isSynthetic: true, lexeme: '"');
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, '"');
+    expect(token.isEof, isTrue);
   }
 
   void test_string_multi_unterminated() {
-    Token token = scan("'''string");
+    Token token = scan("'''string", errorCount: 1);
+    expectError(codeUnterminatedString, 0, 9);
     expectToken(token, TokenType.STRING, 0, 9,
         lexeme: "'''string'''", isSynthetic: true);
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, "'''");
+    expect(token.isEof, isTrue);
   }
 
   void test_string_raw_multi_unterminated() {
-    Token token = scan("r'''string");
+    Token token = scan("r'''string", errorCount: 1);
+    expectError(codeUnterminatedString, 0, 10);
     expectToken(token, TokenType.STRING, 0, 10,
         lexeme: "r'''string'''", isSynthetic: true);
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, "r'''");
+    expect(token.isEof, isTrue);
   }
 
   void test_string_raw_simple_unterminated_eof() {
-    Token token = scan("r'string");
+    Token token = scan("r'string", errorCount: 1);
+    expectError(codeUnterminatedString, 0, 8);
     expectToken(token, TokenType.STRING, 0, 8,
         lexeme: "r'string'", isSynthetic: true);
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, "r'");
+    expect(token.isEof, isTrue);
   }
 
   void test_string_raw_simple_unterminated_eol() {
-    Token token = scan("r'string\n");
+    Token token = scan("r'string\n", errorCount: 1);
+    expectError(codeUnterminatedString, 0, 8);
     expectToken(token, TokenType.STRING, 0, 8,
         lexeme: "r'string'", isSynthetic: true);
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, "r'");
+    expect(token.isEof, isTrue);
   }
 
   void test_string_simple_unterminated_eof() {
-    Token token = scan("'string");
+    Token token = scan("'string", errorCount: 1);
+    expectError(codeUnterminatedString, 0, 7);
     expectToken(token, TokenType.STRING, 0, 7,
         lexeme: "'string'", isSynthetic: true);
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, "'");
+    expect(token.isEof, isTrue);
   }
 
   void test_string_simple_unterminated_eol() {
-    Token token = scan("'string\n");
+    Token token = scan("'string\n", errorCount: 1);
+    expectError(codeUnterminatedString, 0, 7);
+
     expectToken(token, TokenType.STRING, 0, 7,
         lexeme: "'string'", isSynthetic: true);
 
     token = token.next;
-    expect((token as fasta.ErrorToken).errorCode, same(codeUnterminatedString));
-    expect((token as fasta.UnterminatedString).start, "'");
+    expect(token.isEof, isTrue);
   }
 
   void test_match_angle_brackets() {
@@ -720,8 +751,10 @@
       new fasta.StringScanner(source, includeComments: includeComments);
 
   @override
-  Token scan(String source) {
-    final Token first = createScanner(source, includeComments: true).tokenize();
+  Token scan(String source, {int errorCount}) {
+    Scanner scanner = createScanner(source, includeComments: true);
+    scanner.reportErrors = true;
+    final Token first = scanner.tokenize();
     Token token = first;
     while (!token.isEof) {
       Token next = token.next;
@@ -732,6 +765,8 @@
       }
       token = next;
     }
+    scanErrors = scanner.errors;
+    expect(scanErrors, errorCount == null ? isNull : hasLength(errorCount));
     return first;
   }