pkg/front_end/lib/src/fasta/scanner/scanner_main.dart

// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE.md file.

library fasta.scanner.main;

import 'io.dart' show readBytesFromFileSync;

import '../scanner.dart' show ErrorToken, Token, scan;

scanAll(Map<Uri, List<int>> files) {
  Stopwatch sw = new Stopwatch()..start();
  int byteCount = 0;
  files.forEach((Uri uri, List<int> bytes) {
    var token = scan(bytes).tokens;
    if (const bool.fromEnvironment("printTokens")) {
      printTokens(token);
    }
    if (const bool.fromEnvironment('verifyErrorTokens')) {
      verifyErrorTokens(token, uri);
    }
    byteCount += bytes.length - 1;
  });
  sw.stop();
  print("Scanning files took: ${sw.elapsed}");
  print("Bytes/ms: ${byteCount/sw.elapsedMilliseconds}");
}

void printTokens(Token token) {
  while (!token.isEof) {
    print("${token.charOffset}: $token");
    token = token.next;
  }
}

/// Verify that the fasta scanner recovery has moved all of the ErrorTokens
/// to the beginning of the stream. If an out-of-order ErrorToken is
/// found, then print some diagnostic information and throw an exception.
void verifyErrorTokens(Token firstToken, Uri uri) {
  Token token = firstToken;
  while (token is ErrorToken) {
    token = token.next;
  }

  while (!token.isEof) {
    if (token is ErrorToken) {
      print('Found out-of-order ErrorTokens when scanning:\n  $uri');

      // Rescan the token stream up to the error token to find the 10 tokens
      // before the out of order ErrorToken.
      Token errorToken = token;
      Token start = firstToken;
      int count = 0;
      token = firstToken;
      while (token != errorToken) {
        token = token.next;
        if (count < 10) {
          ++count;
        } else {
          start = start.next;
        }
      }

      // Print the out of order error token plus some tokens before and after.
      count = 0;
      token = start;
      while (count < 20 && !token.isEof) {
        print("${token.charOffset}: $token");
        token = token.next;
        ++count;
      }
      throw 'Out of order ErrorToken: $errorToken';
    }
    token = token.next;
  }
}

mainEntryPoint(List<String> arguments) {
  Map<Uri, List<int>> files = <Uri, List<int>>{};
  Stopwatch sw = new Stopwatch()..start();
  for (String name in arguments) {
    Uri uri = Uri.base.resolve(name);
    List<int> bytes = readBytesFromFileSync(uri);
    files[uri] = bytes;
  }
  sw.stop();
  print("Reading files took: ${sw.elapsed}");
  scanAll(files);
}