blob: 50073cd9ba0d8e03359297cbceda84a365701d77 [file] [log] [blame]
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
#include "vm/scanner.h"
#include "platform/assert.h"
#include "vm/dart.h"
#include "vm/flags.h"
#include "vm/object.h"
#include "vm/object_store.h"
#include "vm/symbols.h"
#include "vm/token.h"
#include "vm/unicode.h"
namespace dart {
DEFINE_FLAG(bool, print_tokens, false, "Print scanned tokens.");
// Quick access to the locally defined zone() method.
#define Z (zone())
class ScanContext : public ZoneAllocated {
public:
explicit ScanContext(Scanner* scanner)
: next_(scanner->saved_context_),
string_delimiter_(scanner->string_delimiter_),
string_is_multiline_(scanner->string_is_multiline_),
brace_level_(scanner->brace_level_) {}
void CopyTo(Scanner* scanner) {
scanner->string_delimiter_ = string_delimiter_;
scanner->string_is_multiline_ = string_is_multiline_;
scanner->brace_level_ = brace_level_;
}
ScanContext* next() const { return next_; }
private:
ScanContext* next_;
const char string_delimiter_;
const bool string_is_multiline_;
const int brace_level_;
};
Scanner::KeywordTable Scanner::keywords_[Token::kNumKeywords];
int Scanner::keywords_char_offset_[Scanner::kNumLowercaseChars];
void Scanner::Reset() {
// Non-changing newline properties.
newline_token_.kind = Token::kNEWLINE;
newline_token_.literal = NULL;
// We don't preserve the column information.
newline_token_.position.column = 0;
// Non-changing empty string token properties.
empty_string_token_.kind = Token::kSTRING;
empty_string_token_.literal = &Symbols::Empty();
empty_string_token_.position.column = 0;
lookahead_pos_ = -1;
token_start_ = 0;
c0_ = '\0';
newline_seen_ = false;
prev_token_line_ = 1;
saved_context_ = NULL;
string_delimiter_ = '\0';
string_is_multiline_ = false;
brace_level_ = 0;
c0_pos_.line = 1;
c0_pos_.column = 0;
ReadChar();
}
Scanner::Scanner(const String& src, const String& private_key)
: source_(src),
source_length_(src.Length()),
saved_context_(NULL),
private_key_(String::ZoneHandle(private_key.raw())),
char_at_func_(src.CharAtFunc()),
zone_(Thread::Current()->zone()) {
Reset();
}
Scanner::~Scanner() {}
void Scanner::ErrorMsg(const char* msg) {
current_token_.kind = Token::kERROR;
current_token_.literal = &String::ZoneHandle(Z, Symbols::New(msg));
current_token_.position = c0_pos_;
token_start_ = lookahead_pos_;
current_token_.offset = lookahead_pos_;
}
void Scanner::PushContext() {
ScanContext* ctx = new(Z) ScanContext(this);
saved_context_ = ctx;
string_delimiter_ = '\0';
string_is_multiline_ = false;
brace_level_ = 1; // Account for the opening ${ token.
}
void Scanner::PopContext() {
ASSERT(saved_context_ != NULL);
ASSERT(brace_level_ == 0);
ASSERT(string_delimiter_ == '\0');
ScanContext* ctx = saved_context_;
ctx->CopyTo(this);
saved_context_ = ctx->next();
ASSERT(string_delimiter_ != '\0');
}
void Scanner::BeginStringLiteral(const char delimiter) {
string_delimiter_ = delimiter;
}
void Scanner::EndStringLiteral() {
string_delimiter_ = '\0';
string_is_multiline_ = false;
}
bool Scanner::IsLetter(int32_t c) {
return (('A' <= c) && (c <= 'Z')) || (('a' <= c) && (c <= 'z'));
}
bool Scanner::IsDecimalDigit(int32_t c) {
return '0' <= c && c <= '9';
}
bool Scanner::IsNumberStart(int32_t ch) {
return IsDecimalDigit(ch) || ch == '.';
}
bool Scanner::IsHexDigit(int32_t c) {
return IsDecimalDigit(c)
|| (('A' <= c) && (c <= 'F'))
|| (('a' <= c) && (c <= 'f'));
}
bool Scanner::IsIdentStartChar(int32_t c) {
return IsLetter(c) || (c == '_') || (c == '$');
}
bool Scanner::IsIdentChar(int32_t c) {
return IsLetter(c) || IsDecimalDigit(c) || (c == '_') || (c == '$');
}
bool Scanner::IsIdent(const String& str) {
if (!str.IsOneByteString()) {
return false;
}
if (str.Length() == 0 || !IsIdentStartChar(CallCharAt()(str, 0))) {
return false;
}
for (int i = 1; i < str.Length(); i++) {
if (!IsIdentChar(CallCharAt()(str, i))) {
return false;
}
}
return true;
}
// This method is used when parsing integers and doubles in Dart code. We
// are reusing the Scanner's handling of number literals in that situation.
bool Scanner::IsValidLiteral(const Scanner::GrowableTokenStream& tokens,
Token::Kind literal_kind,
bool* is_positive,
const String** value) {
if ((tokens.length() == 2) &&
(tokens[0].kind == literal_kind) &&
(tokens[1].kind == Token::kEOS)) {
*is_positive = true;
*value = tokens[0].literal;
return true;
}
if ((tokens.length() == 3) &&
((tokens[0].kind == Token::kADD) ||
(tokens[0].kind == Token::kSUB)) &&
(tokens[1].kind == literal_kind) &&
(tokens[2].kind == Token::kEOS)) {
// Check there is no space between "+/-" and number.
if ((tokens[0].offset + 1) != tokens[1].offset) {
return false;
}
*is_positive = tokens[0].kind == Token::kADD;
*value = tokens[1].literal;
return true;
}
return false;
}
void Scanner::ReadChar() {
if (lookahead_pos_ < source_length_) {
if (c0_ == '\n') {
newline_seen_ = true;
c0_pos_.line++;
c0_pos_.column = 0;
if (CallCharAt()(source_, lookahead_pos_) == '\r') {
// Replace a sequence of '\r' '\n' with a single '\n'.
if (LookaheadChar(1) == '\n') {
lookahead_pos_++;
}
}
}
lookahead_pos_++;
c0_pos_.column++;
c0_ = LookaheadChar(0);
// Replace '\r' with '\n'.
if (c0_ == '\r') {
c0_ = '\n';
}
}
}
// Look ahead 'how_many' characters. Returns the character, or '\0' if
// the lookahead position is beyond the end of the string. Does not
// normalize line end characters into '\n'.
int32_t Scanner::LookaheadChar(int how_many) {
ASSERT(how_many >= 0);
int32_t lookahead_char = '\0';
if (lookahead_pos_ + how_many < source_length_) {
lookahead_char = CallCharAt()(source_, lookahead_pos_ + how_many);
}
return lookahead_char;
}
void Scanner::ConsumeWhiteSpace() {
while (c0_ == ' ' || c0_ == '\t' || c0_ == '\n') {
ReadChar();
}
}
void Scanner::ConsumeLineComment() {
ASSERT(c0_ == '/');
while (c0_ != '\n' && c0_ != '\0') {
ReadChar();
}
ReadChar();
current_token_.kind = Token::kWHITESP;
}
void Scanner::ConsumeBlockComment() {
ASSERT(c0_ == '*');
ReadChar();
int nesting_level = 1;
while (true) {
const char c = c0_;
ReadChar();
if (c0_ == '\0') {
break;
}
if (c == '/' && c0_ == '*') {
nesting_level++;
ReadChar(); // Consume asterisk.
} else if (c == '*' && c0_ == '/') {
nesting_level--;
ReadChar(); // Consume slash.
if (nesting_level == 0) {
break;
}
}
}
current_token_.kind =
(nesting_level == 0) ? Token::kWHITESP : Token::kILLEGAL;
}
void Scanner::ScanIdentChars(bool allow_dollar) {
ASSERT(IsIdentStartChar(c0_));
ASSERT(allow_dollar || (c0_ != '$'));
int ident_length = 0;
int ident_pos = lookahead_pos_;
int32_t ident_char0 = CallCharAt()(source_, ident_pos);
while (IsIdentChar(c0_) && (allow_dollar || (c0_ != '$'))) {
ReadChar();
ident_length++;
}
// Check whether the characters we read are a known keyword.
// Note, can't use strcmp since token_chars is not null-terminated.
if (('a' <= ident_char0) && (ident_char0 <= 'z')) {
int i = keywords_char_offset_[ident_char0 - 'a'];
while ((i < Token::kNumKeywords) &&
(keywords_[i].keyword_chars[0] <= ident_char0)) {
if (keywords_[i].keyword_len == ident_length) {
const char* keyword = keywords_[i].keyword_chars;
int char_pos = 1;
while ((char_pos < ident_length) &&
(keyword[char_pos] ==
CallCharAt()(source_, ident_pos + char_pos))) {
char_pos++;
}
if (char_pos == ident_length) {
current_token_.literal = keywords_[i].keyword_symbol;
current_token_.kind = keywords_[i].kind;
return;
}
}
i++;
}
}
// We did not read a keyword.
current_token_.kind = Token::kIDENT;
String& literal =
String::ZoneHandle(Z, Symbols::New(source_, ident_pos, ident_length));
if (ident_char0 == Library::kPrivateIdentifierStart) {
// Private identifiers are mangled on a per library basis.
literal = Symbols::FromConcat(literal, private_key_);
}
current_token_.literal = &literal;
}
// Parse integer or double number literal of format:
// NUMBER = INTEGER | DOUBLE
// INTEGER = D+ | (("0x" | "0X") H+)
// DOUBLE = ((D+ ["." D*]) | ("." D+)) [ EXPONENT ]
// EXPONENT = ("e" | "E") ["+" | "-"] D+
void Scanner::ScanNumber(bool dec_point_seen) {
ASSERT(IsDecimalDigit(c0_));
char first_digit = c0_;
Recognize(dec_point_seen ? Token::kDOUBLE : Token::kINTEGER);
if (!dec_point_seen && first_digit == '0' && (c0_ == 'x' || c0_ == 'X')) {
ReadChar();
if (!IsHexDigit(c0_)) {
ErrorMsg("hexadecimal digit expected");
return;
}
while (IsHexDigit(c0_)) {
ReadChar();
}
} else {
while (IsDecimalDigit(c0_)) {
ReadChar();
}
if (c0_ == '.' && !dec_point_seen && IsDecimalDigit(LookaheadChar(1))) {
Recognize(Token::kDOUBLE);
while (IsDecimalDigit(c0_)) {
ReadChar();
}
}
if (((c0_ == 'e') || (c0_ == 'E')) &&
(IsDecimalDigit(LookaheadChar(1)) ||
(LookaheadChar(1) == '-') ||
(LookaheadChar(1) == '+'))) {
Recognize(Token::kDOUBLE);
if ((c0_ == '-') || (c0_ == '+')) {
ReadChar();
}
if (!IsDecimalDigit(c0_)) {
ErrorMsg("missing exponent digits");
return;
}
while (IsDecimalDigit(c0_)) {
ReadChar();
}
}
}
if (current_token_.kind != Token::kILLEGAL) {
intptr_t len = lookahead_pos_ - token_start_;
const String& str =
String::ZoneHandle(Z, Symbols::New(source_, token_start_, len));
current_token_.literal = &str;
}
}
void Scanner::SkipLine() {
while (c0_ != '\n' && c0_ != '\0') {
ReadChar();
}
}
void Scanner::ScanScriptTag() {
ReadChar();
ASSERT(c0_ == '!');
Recognize(Token::kSCRIPTTAG);
// The script tag extends to the end of the line. Just treat this
// similar to a line comment.
SkipLine();
}
void Scanner::ScanLiteralString(bool is_raw) {
ASSERT(!IsScanningString());
ASSERT(c0_ == '"' || c0_ == '\'');
// Entering string scanning mode.
BeginStringLiteral(c0_);
ReadChar();
if ((c0_ == string_delimiter_) && (LookaheadChar(1) == string_delimiter_)) {
string_is_multiline_ = true;
ReadChar(); // Skip two additional string delimiters.
ReadChar();
}
ScanLiteralStringChars(is_raw, string_is_multiline_);
}
bool Scanner::ScanHexDigits(int digits, int32_t* value) {
*value = 0;
for (int i = 0; i < digits; ++i) {
ReadChar();
if (!IsHexDigit(c0_)) {
ErrorMsg("too few hexadecimal digits");
return false;
}
*value <<= 4;
*value |= Utils::HexDigitToInt(c0_);
}
return true;
}
bool Scanner::ScanHexDigits(int min_digits, int max_digits, int32_t* value) {
*value = 0;
ReadChar();
for (int i = 0; i < max_digits; ++i) {
if (!IsHexDigit(c0_)) {
if (i < min_digits) {
ErrorMsg("hexadecimal digit expected");
return false;
}
break;
}
*value <<= 4;
*value |= Utils::HexDigitToInt(c0_);
ReadChar();
}
return true;
}
void Scanner::ScanEscapedCodePoint(int32_t* code_point) {
ASSERT(c0_ == 'u' || c0_ == 'x');
bool is_valid;
if (c0_ == 'x') {
is_valid = ScanHexDigits(2, code_point);
} else if (c0_ == 'u' && LookaheadChar(1) != '{') {
is_valid = ScanHexDigits(4, code_point);
} else {
ReadChar(); // Skip left curly bracket.
is_valid = ScanHexDigits(1, 6, code_point);
if (is_valid) {
if (c0_ != '}') {
ErrorMsg("expected '}' after character code");
return;
}
}
}
if (is_valid &&
((Utf::IsOutOfRange(*code_point) ||
(Utf16::IsSurrogate(*code_point))))) {
ErrorMsg("invalid code point");
}
}
void Scanner::ScanLiteralStringChars(bool is_raw, bool remove_whitespace) {
GrowableArray<int32_t> string_chars(64);
ASSERT(IsScanningString());
// We are at the first character of a string literal piece. A string literal
// can be broken up into multiple pieces by string interpolation.
while (true) {
if ((c0_ == '\0') || ((c0_ == '\n') && !string_is_multiline_)) {
ErrorMsg("unterminated string literal");
EndStringLiteral();
return;
}
if (c0_ == '\\' && !is_raw) {
// Parse escape sequence.
int32_t escape_char = '\0';
ReadChar();
switch (c0_) {
case 'n':
escape_char = '\n';
break;
case 'r':
escape_char = '\r';
break;
case 'f':
escape_char = '\f';
break;
case 't':
escape_char = '\t';
break;
case 'b':
escape_char = '\b';
break;
case 'v':
escape_char = '\v';
break;
case 'u':
case 'x': {
ScanEscapedCodePoint(&escape_char);
break;
}
default:
if ((c0_ == '\0') || ((c0_ == '\n') && !string_is_multiline_)) {
ErrorMsg("unterminated string literal");
EndStringLiteral();
return;
}
escape_char = c0_;
break;
}
string_chars.Add(escape_char);
} else if (c0_ == '$' && !is_raw) {
// Scanned a string piece.
ASSERT(string_chars.data() != NULL);
// Strings are canonicalized: Allocate a symbol.
current_token_.literal = &String::ZoneHandle(Z,
Symbols::FromUTF32(string_chars.data(), string_chars.length()));
// Preserve error tokens.
if (current_token_.kind != Token::kERROR) {
current_token_.kind = Token::kSTRING;
}
return;
} else if (c0_ == string_delimiter_) {
// Check if we are at the end of the string literal.
if (!string_is_multiline_ ||
((LookaheadChar(1) == string_delimiter_) &&
(LookaheadChar(2) == string_delimiter_))) {
if (string_is_multiline_) {
ReadChar(); // Skip two string delimiters.
ReadChar();
}
// Preserve error tokens.
if (current_token_.kind == Token::kERROR) {
ReadChar();
} else {
Recognize(Token::kSTRING);
ASSERT(string_chars.data() != NULL);
// Strings are canonicalized: Allocate a symbol.
current_token_.literal = &String::ZoneHandle(Z,
Symbols::FromUTF32(string_chars.data(), string_chars.length()));
}
EndStringLiteral();
return;
} else {
string_chars.Add(string_delimiter_);
}
} else {
// Test for a two part utf16 sequence, and decode to a code point
// if we find one.
int32_t ch1 = c0_;
if (Utf16::IsLeadSurrogate(ch1)) {
const int32_t ch2 = LookaheadChar(1);
if (Utf16::IsTrailSurrogate(ch2)) {
ch1 = Utf16::Decode(ch1, ch2);
ReadChar();
}
}
string_chars.Add(ch1);
}
// The first line of a multi-line string is discarded if it only
// contains whitespace.
if (remove_whitespace && (string_chars.Last() == '\n')) {
bool whitespace_only = true;
// Last character is the newline, don't inspect it.
const intptr_t len = string_chars.length() - 1;
for (int i = 0; i < len; i++) {
int32_t ch = string_chars[i];
if ((ch != ' ') && (ch != '\t')) {
// Non-whitespace character, keep the first line.
whitespace_only = false;
break;
}
}
if (whitespace_only) {
string_chars.Clear(); // Discard characters on first line.
}
remove_whitespace = false;
}
ReadChar();
}
}
void Scanner::Scan() {
newline_seen_ = false;
do {
if (!IsScanningString()) {
ConsumeWhiteSpace();
}
token_start_ = lookahead_pos_;
current_token_.offset = lookahead_pos_;
current_token_.position = c0_pos_;
current_token_.literal = NULL;
current_token_.kind = Token::kILLEGAL;
if (IsScanningString()) {
if (c0_ == '$') {
ReadChar(); // Skip the '$' character.
if (IsIdentStartChar(c0_) && (c0_ != '$')) {
ScanIdentNoDollar();
current_token_.kind = Token::kINTERPOL_VAR;
} else if (c0_ == '{') {
Recognize(Token::kINTERPOL_START);
PushContext();
} else {
ErrorMsg("illegal character after $ in string interpolation");
EndStringLiteral();
break;
}
} else {
ScanLiteralStringChars(false, false);
}
break;
}
switch (c0_) {
case '\0':
current_token_.kind = Token::kEOS;
break;
case '+': // + ++ +=
Recognize(Token::kADD);
if (c0_ == '+') {
Recognize(Token::kINCR);
} else if (c0_ == '=') {
Recognize(Token::kASSIGN_ADD);
}
break;
case '-': // - -- -=
Recognize(Token::kSUB);
if (c0_ == '-') {
Recognize(Token::kDECR);
} else if (c0_ == '=') {
Recognize(Token::kASSIGN_SUB);
}
break;
case '*': // * *=
Recognize(Token::kMUL);
if (c0_ == '=') {
Recognize(Token::kASSIGN_MUL);
}
break;
case '%': // % %=
Recognize(Token::kMOD);
if (c0_ == '=') {
Recognize(Token::kASSIGN_MOD);
}
break;
case '/': // / /= // /*
Recognize(Token::kDIV);
if (c0_ == '/') {
ConsumeLineComment();
} else if (c0_ == '*') {
ConsumeBlockComment();
} else if (c0_ == '=') {
Recognize(Token::kASSIGN_DIV);
}
break;
case '&': // & &= &&
Recognize(Token::kBIT_AND);
if (c0_ == '=') {
Recognize(Token::kASSIGN_AND);
} else if (c0_ == '&') {
Recognize(Token::kAND);
}
break;
case '|': // | |= ||
Recognize(Token::kBIT_OR);
if (c0_ == '=') {
Recognize(Token::kASSIGN_OR);
} else if (c0_ == '|') {
Recognize(Token::kOR);
}
break;
case '^': // ^ ^=
Recognize(Token::kBIT_XOR);
if (c0_ == '=') {
Recognize(Token::kASSIGN_XOR);
}
break;
case '[': // [ [] []=
Recognize(Token::kLBRACK);
if (c0_ == ']') {
Recognize(Token::kINDEX);
if (c0_ == '=') {
Recognize(Token::kASSIGN_INDEX);
}
}
break;
case ']': // ]
Recognize(Token::kRBRACK);
break;
case '<': // < <= << <<=
Recognize(Token::kLT);
if (c0_ == '=') {
Recognize(Token::kLTE);
} else if (c0_ == '<') {
Recognize(Token::kSHL);
if (c0_ == '=') {
Recognize(Token::kASSIGN_SHL);
}
}
break;
case '>': // > >= >> >>=
Recognize(Token::kGT);
if (c0_ == '=') {
Recognize(Token::kGTE);
} else if (c0_ == '>') {
Recognize(Token::kSHR);
if (c0_ == '=') {
Recognize(Token::kASSIGN_SHR);
}
}
break;
case '!': // ! !=
Recognize(Token::kNOT);
if (c0_ == '=') {
Recognize(Token::kNE);
}
break;
case '~':
Recognize(Token::kBIT_NOT);
if (c0_ == '/') {
Recognize(Token::kTRUNCDIV);
if (c0_ == '=') {
Recognize(Token::kASSIGN_TRUNCDIV);
}
}
break;
case '=': // = == =>
Recognize(Token::kASSIGN);
if (c0_ == '=') {
Recognize(Token::kEQ);
} else if (c0_ == '>') {
Recognize(Token::kARROW);
}
break;
case '.': // . .. Number
Recognize(Token::kPERIOD);
if (c0_ == '.') {
Recognize(Token::kCASCADE);
} else if (IsDecimalDigit(c0_)) {
ScanNumber(true);
}
break;
case '?': // ? ?. ?? ??=
Recognize(Token::kCONDITIONAL);
if (c0_ == '.') {
Recognize(Token::kQM_PERIOD);
} else if (c0_ == '?') {
Recognize(Token::kIFNULL);
if (c0_ == '=') {
Recognize(Token::kASSIGN_COND);
}
}
break;
case ':':
Recognize(Token::kCOLON);
break;
case ';':
Recognize(Token::kSEMICOLON);
break;
case '{':
Recognize(Token::kLBRACE);
if (IsNestedContext()) {
brace_level_++;
}
break;
case '}':
Recognize(Token::kRBRACE);
if (IsNestedContext()) {
ASSERT(brace_level_ > 0);
brace_level_--;
if (brace_level_ == 0) {
current_token_.kind = Token::kINTERPOL_END;
PopContext();
}
}
break;
case '(':
Recognize(Token::kLPAREN);
break;
case ')':
Recognize(Token::kRPAREN);
break;
case ',':
Recognize(Token::kCOMMA);
break;
case '@':
Recognize(Token::kAT);
break;
case 'r':
if ((LookaheadChar(1) == '"') || (LookaheadChar(1) == '\'')) {
ReadChar();
ScanLiteralString(true);
} else {
ScanIdent();
}
break;
case '"':
case '\'':
ScanLiteralString(false);
break;
case '#':
if (LookaheadChar(1) == '!') {
ScanScriptTag();
} else {
Recognize(Token::kHASH);
}
break;
default:
if (IsIdentStartChar(c0_)) {
ScanIdent();
} else if (IsDecimalDigit(c0_)) {
ScanNumber(false);
} else {
char msg[128];
char utf8_char[5];
int len = Utf8::Encode(c0_, utf8_char);
utf8_char[len] = '\0';
OS::SNPrint(msg, sizeof(msg),
"unexpected character: '%s' (U+%04X)\n", utf8_char, c0_);
ErrorMsg(msg);
ReadChar();
}
}
} while (current_token_.kind == Token::kWHITESP);
}
void Scanner::ScanAll(GrowableTokenStream* token_stream) {
Reset();
do {
Scan();
bool inserted_new_lines = false;
for (intptr_t diff = current_token_.position.line - prev_token_line_;
diff > 0;
diff--) {
newline_token_.position.line = current_token_.position.line - diff;
token_stream->Add(newline_token_);
inserted_new_lines = true;
}
if (inserted_new_lines &&
((current_token_.kind == Token::kINTERPOL_VAR) ||
(current_token_.kind == Token::kINTERPOL_START))) {
// NOTE: If this changes, be sure to update
// Script::GenerateLineNumberArray to stay in sync.
empty_string_token_.position.line = current_token_.position.line;
token_stream->Add(empty_string_token_);
}
token_stream->Add(current_token_);
prev_token_line_ = current_token_.position.line;
} while (current_token_.kind != Token::kEOS);
}
void Scanner::ScanTo(intptr_t token_index) {
int index = 0;
Reset();
do {
Scan();
bool inserted_new_lines = false;
for (intptr_t diff = current_token_.position.line - prev_token_line_;
diff > 0;
diff--) {
index++; // Advance the index to account for tokens added in ScanAll.
inserted_new_lines = true;
}
if (inserted_new_lines &&
((current_token_.kind == Token::kINTERPOL_VAR) ||
(current_token_.kind == Token::kINTERPOL_START))) {
index++; // Advance the index to account for tokens added in ScanAll.
}
index++;
prev_token_line_ = current_token_.position.line;
} while ((token_index >= index) && (current_token_.kind != Token::kEOS));
}
const Scanner::GrowableTokenStream& Scanner::GetStream() {
GrowableTokenStream* ts = new(Z) GrowableTokenStream(128);
ScanAll(ts);
if (FLAG_print_tokens) {
Scanner::PrintTokens(*ts);
}
return *ts;
}
void Scanner::PrintTokens(const GrowableTokenStream& ts) {
int currentLine = -1;
for (int i = 0; i < ts.length(); i++) {
const TokenDescriptor& td = ts[i];
if (currentLine != td.position.line) {
currentLine = td.position.line;
OS::Print("\n%d (%d): ", currentLine, i);
}
OS::Print("%s ", Token::Name(td.kind));
}
OS::Print("\n");
}
void Scanner::InitOnce() {
ASSERT(Isolate::Current() == Dart::vm_isolate());
for (int i = 0; i < kNumLowercaseChars; i++) {
keywords_char_offset_[i] = Token::kNumKeywords;
}
for (int i = 0; i < Token::kNumKeywords; i++) {
Token::Kind token = static_cast<Token::Kind>(Token::kFirstKeyword + i);
keywords_[i].kind = token;
keywords_[i].keyword_chars = Token::Str(token);
keywords_[i].keyword_len = strlen(Token::Str(token));
keywords_[i].keyword_symbol = &Symbols::Keyword(token);
int ch = keywords_[i].keyword_chars[0] - 'a';
if (keywords_char_offset_[ch] == Token::kNumKeywords) {
keywords_char_offset_[ch] = i;
}
}
}
} // namespace dart