blob: fc3bb7e2dc5c4b7cbccf5d91e27753447910369b [file] [log] [blame]
// Copyright (c) 2016, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
/**
* Defines the tokens that are produced by the scanner, used by the parser, and
* referenced from the [AST structure](ast.dart).
*/
import 'dart:collection';
import '../base/syntactic_entity.dart';
import 'string_utilities.dart';
import 'token_constants.dart';
const int NO_PRECEDENCE = 0;
const int ASSIGNMENT_PRECEDENCE = 1;
const int CASCADE_PRECEDENCE = 2;
const int CONDITIONAL_PRECEDENCE = 3;
const int IF_NULL_PRECEDENCE = 4;
const int LOGICAL_OR_PRECEDENCE = 5;
const int LOGICAL_AND_PRECEDENCE = 6;
const int EQUALITY_PRECEDENCE = 7;
const int RELATIONAL_PRECEDENCE = 8;
const int BITWISE_OR_PRECEDENCE = 9;
const int BITWISE_XOR_PRECEDENCE = 10;
const int BITWISE_AND_PRECEDENCE = 11;
const int SHIFT_PRECEDENCE = 12;
const int ADDITIVE_PRECEDENCE = 13;
const int MULTIPLICATIVE_PRECEDENCE = 14;
const int PREFIX_PRECEDENCE = 15;
const int POSTFIX_PRECEDENCE = 16;
const int SELECTOR_PRECEDENCE = 17;
/**
* The opening half of a grouping pair of tokens. This is used for curly
* brackets ('{'), parentheses ('('), and square brackets ('[').
*/
class BeginToken extends SimpleToken {
/**
* The token that corresponds to this token.
*/
Token endToken;
/**
* Initialize a newly created token to have the given [type] at the given
* [offset].
*/
BeginToken(TokenType type, int offset, [CommentToken precedingComment])
: super(type, offset, precedingComment) {
assert(type == TokenType.LT ||
type == TokenType.OPEN_CURLY_BRACKET ||
type == TokenType.OPEN_PAREN ||
type == TokenType.OPEN_SQUARE_BRACKET ||
type == TokenType.QUESTION_PERIOD_OPEN_SQUARE_BRACKET ||
type == TokenType.STRING_INTERPOLATION_EXPRESSION);
}
@override
Token copy() => new BeginToken(type, offset, copyComments(precedingComments));
@override
Token get endGroup => endToken;
/**
* Set the token that corresponds to this token.
*/
set endGroup(Token token) {
endToken = token;
}
}
/**
* A token representing a comment.
*/
class CommentToken extends StringToken {
/**
* The token that contains this comment.
*/
SimpleToken parent;
/**
* Initialize a newly created token to represent a token of the given [type]
* with the given [value] at the given [offset].
*/
CommentToken(TokenType type, String value, int offset)
: super(type, value, offset);
@override
CommentToken copy() => new CommentToken(type, _value, offset);
/**
* Remove this comment token from the list.
*
* This is used when we decide to interpret the comment as syntax.
*/
void remove() {
if (previous != null) {
previous.setNextWithoutSettingPrevious(next);
next?.previous = previous;
} else {
assert(parent.precedingComments == this);
parent.precedingComments = next;
}
}
}
/**
* A documentation comment token.
*/
class DocumentationCommentToken extends CommentToken {
/**
* Initialize a newly created token to represent a token of the given [type]
* with the given [value] at the given [offset].
*/
DocumentationCommentToken(TokenType type, String value, int offset)
: super(type, value, offset);
@override
CommentToken copy() => new DocumentationCommentToken(type, _value, offset);
}
enum KeywordStyle {
reserved,
builtIn,
pseudo,
}
/**
* The keywords in the Dart programming language.
*
* Clients may not extend, implement or mix-in this class.
*/
class Keyword extends TokenType {
static const Keyword ABSTRACT = const Keyword(
"abstract", "ABSTRACT", KeywordStyle.builtIn,
isModifier: true);
static const Keyword AS = const Keyword("as", "AS", KeywordStyle.builtIn,
precedence: RELATIONAL_PRECEDENCE);
static const Keyword ASSERT =
const Keyword("assert", "ASSERT", KeywordStyle.reserved);
static const Keyword ASYNC =
const Keyword("async", "ASYNC", KeywordStyle.pseudo);
static const Keyword AWAIT =
const Keyword("await", "AWAIT", KeywordStyle.pseudo);
static const Keyword BREAK =
const Keyword("break", "BREAK", KeywordStyle.reserved);
static const Keyword CASE =
const Keyword("case", "CASE", KeywordStyle.reserved);
static const Keyword CATCH =
const Keyword("catch", "CATCH", KeywordStyle.reserved);
static const Keyword CLASS = const Keyword(
"class", "CLASS", KeywordStyle.reserved,
isTopLevelKeyword: true);
static const Keyword CONST =
const Keyword("const", "CONST", KeywordStyle.reserved, isModifier: true);
static const Keyword CONTINUE =
const Keyword("continue", "CONTINUE", KeywordStyle.reserved);
static const Keyword COVARIANT = const Keyword(
"covariant", "COVARIANT", KeywordStyle.builtIn,
isModifier: true);
static const Keyword DEFAULT =
const Keyword("default", "DEFAULT", KeywordStyle.reserved);
static const Keyword DEFERRED =
const Keyword("deferred", "DEFERRED", KeywordStyle.builtIn);
static const Keyword DO = const Keyword("do", "DO", KeywordStyle.reserved);
static const Keyword DYNAMIC =
const Keyword("dynamic", "DYNAMIC", KeywordStyle.builtIn);
static const Keyword ELSE =
const Keyword("else", "ELSE", KeywordStyle.reserved);
static const Keyword ENUM = const Keyword(
"enum", "ENUM", KeywordStyle.reserved,
isTopLevelKeyword: true);
static const Keyword EXPORT = const Keyword(
"export", "EXPORT", KeywordStyle.builtIn,
isTopLevelKeyword: true);
static const Keyword EXTENDS =
const Keyword("extends", "EXTENDS", KeywordStyle.reserved);
static const Keyword EXTENSION = const Keyword(
"extension", "EXTENSION", KeywordStyle.builtIn,
isTopLevelKeyword: true);
static const Keyword EXTERNAL = const Keyword(
"external", "EXTERNAL", KeywordStyle.builtIn,
isModifier: true);
static const Keyword FACTORY =
const Keyword("factory", "FACTORY", KeywordStyle.builtIn);
static const Keyword FALSE =
const Keyword("false", "FALSE", KeywordStyle.reserved);
static const Keyword FINAL =
const Keyword("final", "FINAL", KeywordStyle.reserved, isModifier: true);
static const Keyword FINALLY =
const Keyword("finally", "FINALLY", KeywordStyle.reserved);
static const Keyword FOR = const Keyword("for", "FOR", KeywordStyle.reserved);
static const Keyword FUNCTION =
const Keyword("Function", "FUNCTION", KeywordStyle.pseudo);
static const Keyword GET = const Keyword("get", "GET", KeywordStyle.builtIn);
static const Keyword HIDE =
const Keyword("hide", "HIDE", KeywordStyle.pseudo);
static const Keyword IF = const Keyword("if", "IF", KeywordStyle.reserved);
static const Keyword IMPLEMENTS =
const Keyword("implements", "IMPLEMENTS", KeywordStyle.builtIn);
static const Keyword IMPORT = const Keyword(
"import", "IMPORT", KeywordStyle.builtIn,
isTopLevelKeyword: true);
static const Keyword IN = const Keyword("in", "IN", KeywordStyle.reserved);
static const Keyword INOUT =
const Keyword("inout", "INOUT", KeywordStyle.pseudo);
static const Keyword INTERFACE =
const Keyword("interface", "INTERFACE", KeywordStyle.builtIn);
static const Keyword IS = const Keyword("is", "IS", KeywordStyle.reserved,
precedence: RELATIONAL_PRECEDENCE);
static const Keyword LATE =
const Keyword("late", "LATE", KeywordStyle.builtIn, isModifier: true);
static const Keyword LIBRARY = const Keyword(
"library", "LIBRARY", KeywordStyle.builtIn,
isTopLevelKeyword: true);
static const Keyword MIXIN = const Keyword(
"mixin", "MIXIN", KeywordStyle.builtIn,
isTopLevelKeyword: true);
static const Keyword NATIVE =
const Keyword("native", "NATIVE", KeywordStyle.pseudo);
static const Keyword NEW = const Keyword("new", "NEW", KeywordStyle.reserved);
static const Keyword NULL =
const Keyword("null", "NULL", KeywordStyle.reserved);
static const Keyword OF = const Keyword("of", "OF", KeywordStyle.pseudo);
static const Keyword ON = const Keyword("on", "ON", KeywordStyle.pseudo);
static const Keyword OPERATOR =
const Keyword("operator", "OPERATOR", KeywordStyle.builtIn);
static const Keyword OUT = const Keyword("out", "OUT", KeywordStyle.pseudo);
static const Keyword PART = const Keyword(
"part", "PART", KeywordStyle.builtIn,
isTopLevelKeyword: true);
static const Keyword PATCH =
const Keyword("patch", "PATCH", KeywordStyle.pseudo);
static const Keyword REQUIRED = const Keyword(
"required", "REQUIRED", KeywordStyle.builtIn,
isModifier: true);
static const Keyword RETHROW =
const Keyword("rethrow", "RETHROW", KeywordStyle.reserved);
static const Keyword RETURN =
const Keyword("return", "RETURN", KeywordStyle.reserved);
static const Keyword SET = const Keyword("set", "SET", KeywordStyle.builtIn);
static const Keyword SHOW =
const Keyword("show", "SHOW", KeywordStyle.pseudo);
static const Keyword SOURCE =
const Keyword("source", "SOURCE", KeywordStyle.pseudo);
static const Keyword STATIC =
const Keyword("static", "STATIC", KeywordStyle.builtIn, isModifier: true);
static const Keyword SUPER =
const Keyword("super", "SUPER", KeywordStyle.reserved);
static const Keyword SWITCH =
const Keyword("switch", "SWITCH", KeywordStyle.reserved);
static const Keyword SYNC =
const Keyword("sync", "SYNC", KeywordStyle.pseudo);
static const Keyword THIS =
const Keyword("this", "THIS", KeywordStyle.reserved);
static const Keyword THROW =
const Keyword("throw", "THROW", KeywordStyle.reserved);
static const Keyword TRUE =
const Keyword("true", "TRUE", KeywordStyle.reserved);
static const Keyword TRY = const Keyword("try", "TRY", KeywordStyle.reserved);
static const Keyword TYPEDEF = const Keyword(
"typedef", "TYPEDEF", KeywordStyle.builtIn,
isTopLevelKeyword: true);
static const Keyword VAR =
const Keyword("var", "VAR", KeywordStyle.reserved, isModifier: true);
static const Keyword VOID =
const Keyword("void", "VOID", KeywordStyle.reserved);
static const Keyword WHILE =
const Keyword("while", "WHILE", KeywordStyle.reserved);
static const Keyword WITH =
const Keyword("with", "WITH", KeywordStyle.reserved);
static const Keyword YIELD =
const Keyword("yield", "YIELD", KeywordStyle.pseudo);
static const List<Keyword> values = const <Keyword>[
ABSTRACT,
AS,
ASSERT,
ASYNC,
AWAIT,
BREAK,
CASE,
CATCH,
CLASS,
CONST,
CONTINUE,
COVARIANT,
DEFAULT,
DEFERRED,
DO,
DYNAMIC,
ELSE,
ENUM,
EXPORT,
EXTENDS,
EXTENSION,
EXTERNAL,
FACTORY,
FALSE,
FINAL,
FINALLY,
FOR,
FUNCTION,
GET,
HIDE,
IF,
IMPLEMENTS,
IMPORT,
IN,
INOUT,
INTERFACE,
IS,
LATE,
LIBRARY,
MIXIN,
NATIVE,
NEW,
NULL,
OF,
ON,
OPERATOR,
OUT,
PART,
PATCH,
REQUIRED,
RETHROW,
RETURN,
SET,
SHOW,
SOURCE,
STATIC,
SUPER,
SWITCH,
SYNC,
THIS,
THROW,
TRUE,
TRY,
TYPEDEF,
VAR,
VOID,
WHILE,
WITH,
YIELD,
];
/**
* A table mapping the lexemes of keywords to the corresponding keyword.
*/
static final Map<String, Keyword> keywords = _createKeywordMap();
final KeywordStyle keywordStyle;
/**
* Initialize a newly created keyword.
*/
const Keyword(String lexeme, String name, this.keywordStyle,
{bool isModifier: false,
bool isTopLevelKeyword: false,
int precedence: NO_PRECEDENCE})
: super(lexeme, name, precedence, KEYWORD_TOKEN,
isModifier: isModifier, isTopLevelKeyword: isTopLevelKeyword);
@override
bool get isBuiltIn => keywordStyle == KeywordStyle.builtIn;
@override
bool get isPseudo => keywordStyle == KeywordStyle.pseudo;
bool get isBuiltInOrPseudo => isBuiltIn || isPseudo;
@override
bool get isReservedWord => keywordStyle == KeywordStyle.reserved;
/**
* The name of the keyword type.
*/
String get name => lexeme.toUpperCase();
@override
String toString() => name;
/**
* Create a table mapping the lexemes of keywords to the corresponding keyword
* and return the table that was created.
*/
static Map<String, Keyword> _createKeywordMap() {
LinkedHashMap<String, Keyword> result =
new LinkedHashMap<String, Keyword>();
for (Keyword keyword in values) {
result[keyword.lexeme] = keyword;
}
return result;
}
}
/**
* A token representing a keyword in the language.
*/
class KeywordToken extends SimpleToken {
@override
final Keyword keyword;
/**
* Initialize a newly created token to represent the given [keyword] at the
* given [offset].
*/
KeywordToken(this.keyword, int offset, [CommentToken precedingComment])
: super(keyword, offset, precedingComment);
@override
Token copy() =>
new KeywordToken(keyword, offset, copyComments(precedingComments));
@override
bool get isIdentifier => keyword.isPseudo || keyword.isBuiltIn;
@override
bool get isKeyword => true;
@override
bool get isKeywordOrIdentifier => true;
@override
Object value() => keyword;
}
/**
* A specialized comment token representing a language version
* (e.g. '// @dart = 2.1').
*/
class LanguageVersionToken extends CommentToken {
/**
* The major language version.
*/
final int major;
/**
* The minor language version.
*/
final int minor;
LanguageVersionToken.from(String text, int offset, this.major, this.minor)
: super(TokenType.SINGLE_LINE_COMMENT, text, offset);
@override
LanguageVersionToken copy() =>
new LanguageVersionToken.from(lexeme, offset, major, minor);
}
/**
* A token that was scanned from the input. Each token knows which tokens
* precede and follow it, acting as a link in a doubly linked list of tokens.
*/
class SimpleToken implements Token {
/**
* The type of the token.
*/
@override
final TokenType type;
/**
* The offset from the beginning of the file to the first character in the
* token.
*/
@override
int offset = 0;
/**
* The previous token in the token stream.
*/
@override
Token previous;
@override
Token next;
/**
* The first comment in the list of comments that precede this token.
*/
CommentToken _precedingComment;
/**
* Initialize a newly created token to have the given [type] and [offset].
*/
SimpleToken(this.type, this.offset, [this._precedingComment]) {
_setCommentParent(_precedingComment);
}
@override
int get charCount => length;
@override
int get charOffset => offset;
@override
int get charEnd => end;
@override
Token get beforeSynthetic => null;
@override
set beforeSynthetic(Token previous) {
// ignored
}
@override
int get end => offset + length;
@override
Token get endGroup => null;
@override
bool get isEof => type == TokenType.EOF;
@override
bool get isIdentifier => false;
@override
bool get isKeyword => false;
@override
bool get isKeywordOrIdentifier => isIdentifier;
@override
bool get isModifier => type.isModifier;
@override
bool get isOperator => type.isOperator;
@override
bool get isSynthetic => length == 0;
@override
bool get isTopLevelKeyword => type.isTopLevelKeyword;
@override
bool get isUserDefinableOperator => type.isUserDefinableOperator;
@override
Keyword get keyword => null;
@override
int get kind => type.kind;
@override
int get length => lexeme.length;
@override
String get lexeme => type.lexeme;
@override
CommentToken get precedingComments => _precedingComment;
void set precedingComments(CommentToken comment) {
_precedingComment = comment;
_setCommentParent(_precedingComment);
}
@override
String get stringValue => type.stringValue;
@override
Token copy() =>
new SimpleToken(type, offset, copyComments(precedingComments));
@override
Token copyComments(Token token) {
if (token == null) {
return null;
}
Token head = token.copy();
Token tail = head;
token = token.next;
while (token != null) {
tail = tail.setNext(token.copy());
token = token.next;
}
return head;
}
@override
bool matchesAny(List<TokenType> types) {
for (TokenType type in types) {
if (this.type == type) {
return true;
}
}
return false;
}
@override
Token setNext(Token token) {
next = token;
token.previous = this;
token.beforeSynthetic = this;
return token;
}
@override
Token setNextWithoutSettingPrevious(Token token) {
next = token;
return token;
}
@override
String toString() => lexeme;
@override
Object value() => lexeme;
/**
* Sets the `parent` property to `this` for the given [comment] and all the
* next tokens.
*/
void _setCommentParent(CommentToken comment) {
while (comment != null) {
comment.parent = this;
comment = comment.next;
}
}
}
/**
* A token whose value is independent of it's type.
*/
class StringToken extends SimpleToken {
/**
* The lexeme represented by this token.
*/
String _value;
/**
* Initialize a newly created token to represent a token of the given [type]
* with the given [value] at the given [offset].
*/
StringToken(TokenType type, String value, int offset,
[CommentToken precedingComment])
: super(type, offset, precedingComment) {
this._value = StringUtilities.intern(value);
}
@override
bool get isIdentifier => identical(kind, IDENTIFIER_TOKEN);
@override
String get lexeme => _value;
@override
Token copy() =>
new StringToken(type, _value, offset, copyComments(precedingComments));
@override
String value() => _value;
}
/**
* A synthetic begin token.
*/
class SyntheticBeginToken extends BeginToken {
/**
* Initialize a newly created token to have the given [type] at the given
* [offset].
*/
SyntheticBeginToken(TokenType type, int offset,
[CommentToken precedingComment])
: super(type, offset, precedingComment);
@override
Token copy() =>
new SyntheticBeginToken(type, offset, copyComments(precedingComments));
@override
bool get isSynthetic => true;
@override
int get length => 0;
}
/**
* A synthetic keyword token.
*/
class SyntheticKeywordToken extends KeywordToken {
/**
* Initialize a newly created token to represent the given [keyword] at the
* given [offset].
*/
SyntheticKeywordToken(Keyword keyword, int offset) : super(keyword, offset);
@override
int get length => 0;
@override
Token copy() => new SyntheticKeywordToken(keyword, offset);
}
/**
* A token whose value is independent of it's type.
*/
class SyntheticStringToken extends StringToken {
final int _length;
/**
* Initialize a newly created token to represent a token of the given [type]
* with the given [value] at the given [offset]. If the [length] is
* not specified, then it defaults to the length of [value].
*/
SyntheticStringToken(TokenType type, String value, int offset, [this._length])
: super(type, value, offset);
@override
bool get isSynthetic => true;
@override
int get length => _length ?? super.length;
@override
Token copy() => new SyntheticStringToken(type, _value, offset, _length);
}
/**
* A synthetic token.
*/
class SyntheticToken extends SimpleToken {
SyntheticToken(TokenType type, int offset) : super(type, offset);
@override
Token beforeSynthetic;
@override
bool get isSynthetic => true;
@override
int get length => 0;
@override
Token copy() => new SyntheticToken(type, offset);
}
/**
* A token that was scanned from the input. Each token knows which tokens
* precede and follow it, acting as a link in a doubly linked list of tokens.
*
* Clients may not extend, implement or mix-in this class.
*/
abstract class Token implements SyntacticEntity {
/**
* Initialize a newly created token to have the given [type] and [offset].
*/
factory Token(TokenType type, int offset, [CommentToken preceedingComment]) =
SimpleToken;
/**
* Initialize a newly created end-of-file token to have the given [offset].
*/
factory Token.eof(int offset, [CommentToken precedingComments]) {
Token eof = new SimpleToken(TokenType.EOF, offset, precedingComments);
// EOF points to itself so there's always infinite look-ahead.
eof.previous = eof;
eof.next = eof;
return eof;
}
/**
* The number of characters parsed by this token.
*/
int get charCount;
/**
* The character offset of the start of this token within the source text.
*/
int get charOffset;
/**
* The character offset of the end of this token within the source text.
*/
int get charEnd;
/**
* The token before this synthetic token,
* or `null` if this is not a synthetic `)`, `]`, `}`, or `>` token.
*/
Token get beforeSynthetic;
/**
* Set token before this synthetic `)`, `]`, `}`, or `>` token,
* and ignored otherwise.
*/
set beforeSynthetic(Token previous);
@override
int get end;
/**
* The token that corresponds to this token, or `null` if this token is not
* the first of a pair of matching tokens (such as parentheses).
*/
Token get endGroup => null;
/**
* Return `true` if this token represents an end of file.
*/
bool get isEof;
/**
* True if this token is an identifier. Some keywords allowed as identifiers,
* see implementation in [KeywordToken].
*/
bool get isIdentifier;
/**
* True if this token is a keyword. Some keywords allowed as identifiers,
* see implementation in [KeywordToken].
*/
bool get isKeyword;
/**
* True if this token is a keyword or an identifier.
*/
bool get isKeywordOrIdentifier;
/**
* Return `true` if this token is a modifier such as `abstract` or `const`.
*/
bool get isModifier;
/**
* Return `true` if this token represents an operator.
*/
bool get isOperator;
/**
* Return `true` if this token is a synthetic token. A synthetic token is a
* token that was introduced by the parser in order to recover from an error
* in the code.
*/
bool get isSynthetic;
/**
* Return `true` if this token is a keyword starting a top level declaration
* such as `class`, `enum`, `import`, etc.
*/
bool get isTopLevelKeyword;
/**
* Return `true` if this token represents an operator that can be defined by
* users.
*/
bool get isUserDefinableOperator;
/**
* Return the keyword, if a keyword token, or `null` otherwise.
*/
Keyword get keyword;
/**
* The kind enum of this token as determined by its [type].
*/
int get kind;
@override
int get length;
/**
* Return the lexeme that represents this token.
*
* For [StringToken]s the [lexeme] includes the quotes, explicit escapes, etc.
*/
String get lexeme;
/**
* Return the next token in the token stream.
*/
Token get next;
/**
* Return the next token in the token stream.
*/
void set next(Token next);
@override
int get offset;
/**
* Set the offset from the beginning of the file to the first character in
* the token to the given [offset].
*/
void set offset(int offset);
/**
* Return the first comment in the list of comments that precede this token,
* or `null` if there are no comments preceding this token. Additional
* comments can be reached by following the token stream using [next] until
* `null` is returned.
*
* For example, if the original contents were `/* one */ /* two */ id`, then
* the first preceding comment token will have a lexeme of `/* one */` and
* the next comment token will have a lexeme of `/* two */`.
*/
CommentToken get precedingComments;
/**
* Return the previous token in the token stream.
*/
Token get previous;
/**
* Set the previous token in the token stream to the given [token].
*/
void set previous(Token token);
/**
* For symbol and keyword tokens, returns the string value represented by this
* token. For [StringToken]s this method returns [:null:].
*
* For [SymbolToken]s and [KeywordToken]s, the string value is a compile-time
* constant originating in the [TokenType] or in the [Keyword] instance.
* This allows testing for keywords and symbols using [:identical:], e.g.,
* [:identical('class', token.value):].
*
* Note that returning [:null:] for string tokens is important to identify
* symbols and keywords, we cannot use [lexeme] instead. The string literal
* "$a($b"
* produces ..., SymbolToken($), StringToken(a), StringToken((), ...
*
* After parsing the identifier 'a', the parser tests for a function
* declaration using [:identical(next.stringValue, '('):], which (rightfully)
* returns false because stringValue returns [:null:].
*/
String get stringValue;
/**
* Return the type of the token.
*/
TokenType get type;
/**
* Return a newly created token that is a copy of this tokens
* including any [preceedingComment] tokens,
* but that is not a part of any token stream.
*/
Token copy();
/**
* Copy a linked list of comment tokens identical to the given comment tokens.
*/
Token copyComments(Token token);
/**
* Return `true` if this token has any one of the given [types].
*/
bool matchesAny(List<TokenType> types);
/**
* Set the next token in the token stream to the given [token]. This has the
* side-effect of setting this token to be the previous token for the given
* token. Return the token that was passed in.
*/
Token setNext(Token token);
/**
* Set the next token in the token stream to the given token without changing
* which token is the previous token for the given token. Return the token
* that was passed in.
*/
Token setNextWithoutSettingPrevious(Token token);
/**
* Returns a textual representation of this token to be used for debugging
* purposes. The resulting string might contain information about the
* structure of the token, for example 'StringToken(foo)' for the identifier
* token 'foo'.
*
* Use [lexeme] for the text actually parsed by the token.
*/
@override
String toString();
/**
* Return the value of this token. For keyword tokens, this is the keyword
* associated with the token, for other tokens it is the lexeme associated
* with the token.
*/
Object value();
/**
* Compare the given [tokens] to find the token that appears first in the
* source being parsed. That is, return the left-most of all of the tokens.
* The list must be non-`null`, but the elements of the list are allowed to be
* `null`. Return the token with the smallest offset, or `null` if the list is
* empty or if all of the elements of the list are `null`.
*/
static Token lexicallyFirst(List<Token> tokens) {
Token first = null;
int offset = -1;
int length = tokens.length;
for (int i = 0; i < length; i++) {
Token token = tokens[i];
if (token != null && (offset < 0 || token.offset < offset)) {
first = token;
offset = token.offset;
}
}
return first;
}
}
/**
* The classes (or groups) of tokens with a similar use.
*/
class TokenClass {
/**
* A value used to indicate that the token type is not part of any specific
* class of token.
*/
static const TokenClass NO_CLASS = const TokenClass('NO_CLASS');
/**
* A value used to indicate that the token type is an additive operator.
*/
static const TokenClass ADDITIVE_OPERATOR =
const TokenClass('ADDITIVE_OPERATOR', ADDITIVE_PRECEDENCE);
/**
* A value used to indicate that the token type is an assignment operator.
*/
static const TokenClass ASSIGNMENT_OPERATOR =
const TokenClass('ASSIGNMENT_OPERATOR', ASSIGNMENT_PRECEDENCE);
/**
* A value used to indicate that the token type is a bitwise-and operator.
*/
static const TokenClass BITWISE_AND_OPERATOR =
const TokenClass('BITWISE_AND_OPERATOR', BITWISE_AND_PRECEDENCE);
/**
* A value used to indicate that the token type is a bitwise-or operator.
*/
static const TokenClass BITWISE_OR_OPERATOR =
const TokenClass('BITWISE_OR_OPERATOR', BITWISE_OR_PRECEDENCE);
/**
* A value used to indicate that the token type is a bitwise-xor operator.
*/
static const TokenClass BITWISE_XOR_OPERATOR =
const TokenClass('BITWISE_XOR_OPERATOR', BITWISE_XOR_PRECEDENCE);
/**
* A value used to indicate that the token type is a cascade operator.
*/
static const TokenClass CASCADE_OPERATOR =
const TokenClass('CASCADE_OPERATOR', CASCADE_PRECEDENCE);
/**
* A value used to indicate that the token type is a conditional operator.
*/
static const TokenClass CONDITIONAL_OPERATOR =
const TokenClass('CONDITIONAL_OPERATOR', CONDITIONAL_PRECEDENCE);
/**
* A value used to indicate that the token type is an equality operator.
*/
static const TokenClass EQUALITY_OPERATOR =
const TokenClass('EQUALITY_OPERATOR', EQUALITY_PRECEDENCE);
/**
* A value used to indicate that the token type is an if-null operator.
*/
static const TokenClass IF_NULL_OPERATOR =
const TokenClass('IF_NULL_OPERATOR', IF_NULL_PRECEDENCE);
/**
* A value used to indicate that the token type is a logical-and operator.
*/
static const TokenClass LOGICAL_AND_OPERATOR =
const TokenClass('LOGICAL_AND_OPERATOR', LOGICAL_AND_PRECEDENCE);
/**
* A value used to indicate that the token type is a logical-or operator.
*/
static const TokenClass LOGICAL_OR_OPERATOR =
const TokenClass('LOGICAL_OR_OPERATOR', LOGICAL_OR_PRECEDENCE);
/**
* A value used to indicate that the token type is a multiplicative operator.
*/
static const TokenClass MULTIPLICATIVE_OPERATOR =
const TokenClass('MULTIPLICATIVE_OPERATOR', MULTIPLICATIVE_PRECEDENCE);
/**
* A value used to indicate that the token type is a relational operator.
*/
static const TokenClass RELATIONAL_OPERATOR =
const TokenClass('RELATIONAL_OPERATOR', RELATIONAL_PRECEDENCE);
/**
* A value used to indicate that the token type is a shift operator.
*/
static const TokenClass SHIFT_OPERATOR =
const TokenClass('SHIFT_OPERATOR', SHIFT_PRECEDENCE);
/**
* A value used to indicate that the token type is a unary operator.
*/
static const TokenClass UNARY_POSTFIX_OPERATOR =
const TokenClass('UNARY_POSTFIX_OPERATOR', POSTFIX_PRECEDENCE);
/**
* A value used to indicate that the token type is a unary operator.
*/
static const TokenClass UNARY_PREFIX_OPERATOR =
const TokenClass('UNARY_PREFIX_OPERATOR', PREFIX_PRECEDENCE);
/**
* The name of the token class.
*/
final String name;
/**
* The precedence of tokens of this class, or `0` if the such tokens do not
* represent an operator.
*/
final int precedence;
/**
* Initialize a newly created class of tokens to have the given [name] and
* [precedence].
*/
const TokenClass(this.name, [this.precedence = NO_PRECEDENCE]);
@override
String toString() => name;
}
/**
* The types of tokens that can be returned by the scanner.
*
* Clients may not extend, implement or mix-in this class.
*/
class TokenType {
/**
* The type of the token that marks the start or end of the input.
*/
static const TokenType EOF =
const TokenType('', 'EOF', NO_PRECEDENCE, EOF_TOKEN);
static const TokenType DOUBLE = const TokenType(
'double', 'DOUBLE', NO_PRECEDENCE, DOUBLE_TOKEN,
stringValue: null);
static const TokenType HEXADECIMAL = const TokenType(
'hexadecimal', 'HEXADECIMAL', NO_PRECEDENCE, HEXADECIMAL_TOKEN,
stringValue: null);
static const TokenType IDENTIFIER = const TokenType(
'identifier', 'IDENTIFIER', NO_PRECEDENCE, IDENTIFIER_TOKEN,
stringValue: null);
static const TokenType INT = const TokenType(
'int', 'INT', NO_PRECEDENCE, INT_TOKEN,
stringValue: null);
static const TokenType MULTI_LINE_COMMENT = const TokenType(
'comment', 'MULTI_LINE_COMMENT', NO_PRECEDENCE, COMMENT_TOKEN,
stringValue: null);
static const TokenType SCRIPT_TAG =
const TokenType('script', 'SCRIPT_TAG', NO_PRECEDENCE, SCRIPT_TOKEN);
static const TokenType SINGLE_LINE_COMMENT = const TokenType(
'comment', 'SINGLE_LINE_COMMENT', NO_PRECEDENCE, COMMENT_TOKEN,
stringValue: null);
static const TokenType STRING = const TokenType(
'string', 'STRING', NO_PRECEDENCE, STRING_TOKEN,
stringValue: null);
static const TokenType AMPERSAND = const TokenType(
'&', 'AMPERSAND', BITWISE_AND_PRECEDENCE, AMPERSAND_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType AMPERSAND_AMPERSAND = const TokenType('&&',
'AMPERSAND_AMPERSAND', LOGICAL_AND_PRECEDENCE, AMPERSAND_AMPERSAND_TOKEN,
isOperator: true);
// This is not yet part of the language and not supported by fasta
static const TokenType AMPERSAND_AMPERSAND_EQ = const TokenType(
'&&=',
'AMPERSAND_AMPERSAND_EQ',
ASSIGNMENT_PRECEDENCE,
AMPERSAND_AMPERSAND_EQ_TOKEN,
isOperator: true);
static const TokenType AMPERSAND_EQ = const TokenType(
'&=', 'AMPERSAND_EQ', ASSIGNMENT_PRECEDENCE, AMPERSAND_EQ_TOKEN,
isOperator: true);
static const TokenType AT =
const TokenType('@', 'AT', NO_PRECEDENCE, AT_TOKEN);
static const TokenType BANG = const TokenType(
'!', 'BANG', PREFIX_PRECEDENCE, BANG_TOKEN,
isOperator: true);
static const TokenType BANG_EQ = const TokenType(
'!=', 'BANG_EQ', EQUALITY_PRECEDENCE, BANG_EQ_TOKEN,
isOperator: true);
static const TokenType BANG_EQ_EQ = const TokenType(
'!==', 'BANG_EQ_EQ', EQUALITY_PRECEDENCE, BANG_EQ_EQ_TOKEN);
static const TokenType BAR = const TokenType(
'|', 'BAR', BITWISE_OR_PRECEDENCE, BAR_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType BAR_BAR = const TokenType(
'||', 'BAR_BAR', LOGICAL_OR_PRECEDENCE, BAR_BAR_TOKEN,
isOperator: true);
// This is not yet part of the language and not supported by fasta
static const TokenType BAR_BAR_EQ = const TokenType(
'||=', 'BAR_BAR_EQ', ASSIGNMENT_PRECEDENCE, BAR_BAR_EQ_TOKEN,
isOperator: true);
static const TokenType BAR_EQ = const TokenType(
'|=', 'BAR_EQ', ASSIGNMENT_PRECEDENCE, BAR_EQ_TOKEN,
isOperator: true);
static const TokenType COLON =
const TokenType(':', 'COLON', NO_PRECEDENCE, COLON_TOKEN);
static const TokenType COMMA =
const TokenType(',', 'COMMA', NO_PRECEDENCE, COMMA_TOKEN);
static const TokenType CARET = const TokenType(
'^', 'CARET', BITWISE_XOR_PRECEDENCE, CARET_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType CARET_EQ = const TokenType(
'^=', 'CARET_EQ', ASSIGNMENT_PRECEDENCE, CARET_EQ_TOKEN,
isOperator: true);
static const TokenType CLOSE_CURLY_BRACKET = const TokenType(
'}', 'CLOSE_CURLY_BRACKET', NO_PRECEDENCE, CLOSE_CURLY_BRACKET_TOKEN);
static const TokenType CLOSE_PAREN =
const TokenType(')', 'CLOSE_PAREN', NO_PRECEDENCE, CLOSE_PAREN_TOKEN);
static const TokenType CLOSE_SQUARE_BRACKET = const TokenType(
']', 'CLOSE_SQUARE_BRACKET', NO_PRECEDENCE, CLOSE_SQUARE_BRACKET_TOKEN);
static const TokenType EQ = const TokenType(
'=', 'EQ', ASSIGNMENT_PRECEDENCE, EQ_TOKEN,
isOperator: true);
static const TokenType EQ_EQ = const TokenType(
'==', 'EQ_EQ', EQUALITY_PRECEDENCE, EQ_EQ_TOKEN,
isOperator: true, isUserDefinableOperator: true);
/// The `===` operator is not supported in the Dart language
/// but is parsed as such by the scanner to support better recovery
/// when a JavaScript code snippet is pasted into a Dart file.
static const TokenType EQ_EQ_EQ =
const TokenType('===', 'EQ_EQ_EQ', EQUALITY_PRECEDENCE, EQ_EQ_EQ_TOKEN);
static const TokenType FUNCTION =
const TokenType('=>', 'FUNCTION', NO_PRECEDENCE, FUNCTION_TOKEN);
static const TokenType GT = const TokenType(
'>', 'GT', RELATIONAL_PRECEDENCE, GT_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType GT_EQ = const TokenType(
'>=', 'GT_EQ', RELATIONAL_PRECEDENCE, GT_EQ_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType GT_GT = const TokenType(
'>>', 'GT_GT', SHIFT_PRECEDENCE, GT_GT_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType GT_GT_EQ = const TokenType(
'>>=', 'GT_GT_EQ', ASSIGNMENT_PRECEDENCE, GT_GT_EQ_TOKEN,
isOperator: true);
static const TokenType GT_GT_GT = const TokenType(
'>>>', 'GT_GT_GT', SHIFT_PRECEDENCE, GT_GT_GT_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType GT_GT_GT_EQ = const TokenType(
'>>>=', 'GT_GT_GT_EQ', ASSIGNMENT_PRECEDENCE, GT_GT_GT_EQ_TOKEN,
isOperator: true);
static const TokenType HASH =
const TokenType('#', 'HASH', NO_PRECEDENCE, HASH_TOKEN);
static const TokenType INDEX = const TokenType(
'[]', 'INDEX', SELECTOR_PRECEDENCE, INDEX_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType INDEX_EQ = const TokenType(
'[]=', 'INDEX_EQ', NO_PRECEDENCE, INDEX_EQ_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType LT = const TokenType(
'<', 'LT', RELATIONAL_PRECEDENCE, LT_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType LT_EQ = const TokenType(
'<=', 'LT_EQ', RELATIONAL_PRECEDENCE, LT_EQ_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType LT_LT = const TokenType(
'<<', 'LT_LT', SHIFT_PRECEDENCE, LT_LT_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType LT_LT_EQ = const TokenType(
'<<=', 'LT_LT_EQ', ASSIGNMENT_PRECEDENCE, LT_LT_EQ_TOKEN,
isOperator: true);
static const TokenType MINUS = const TokenType(
'-', 'MINUS', ADDITIVE_PRECEDENCE, MINUS_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType MINUS_EQ = const TokenType(
'-=', 'MINUS_EQ', ASSIGNMENT_PRECEDENCE, MINUS_EQ_TOKEN,
isOperator: true);
static const TokenType MINUS_MINUS = const TokenType(
'--', 'MINUS_MINUS', POSTFIX_PRECEDENCE, MINUS_MINUS_TOKEN,
isOperator: true);
static const TokenType OPEN_CURLY_BRACKET = const TokenType(
'{', 'OPEN_CURLY_BRACKET', NO_PRECEDENCE, OPEN_CURLY_BRACKET_TOKEN);
static const TokenType OPEN_PAREN =
const TokenType('(', 'OPEN_PAREN', SELECTOR_PRECEDENCE, OPEN_PAREN_TOKEN);
static const TokenType OPEN_SQUARE_BRACKET = const TokenType('[',
'OPEN_SQUARE_BRACKET', SELECTOR_PRECEDENCE, OPEN_SQUARE_BRACKET_TOKEN);
static const TokenType PERCENT = const TokenType(
'%', 'PERCENT', MULTIPLICATIVE_PRECEDENCE, PERCENT_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType PERCENT_EQ = const TokenType(
'%=', 'PERCENT_EQ', ASSIGNMENT_PRECEDENCE, PERCENT_EQ_TOKEN,
isOperator: true);
static const TokenType PERIOD =
const TokenType('.', 'PERIOD', SELECTOR_PRECEDENCE, PERIOD_TOKEN);
static const TokenType PERIOD_PERIOD = const TokenType(
'..', 'PERIOD_PERIOD', CASCADE_PRECEDENCE, PERIOD_PERIOD_TOKEN,
isOperator: true);
static const TokenType PLUS = const TokenType(
'+', 'PLUS', ADDITIVE_PRECEDENCE, PLUS_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType PLUS_EQ = const TokenType(
'+=', 'PLUS_EQ', ASSIGNMENT_PRECEDENCE, PLUS_EQ_TOKEN,
isOperator: true);
static const TokenType PLUS_PLUS = const TokenType(
'++', 'PLUS_PLUS', POSTFIX_PRECEDENCE, PLUS_PLUS_TOKEN,
isOperator: true);
static const TokenType QUESTION = const TokenType(
'?', 'QUESTION', CONDITIONAL_PRECEDENCE, QUESTION_TOKEN,
isOperator: true);
static const TokenType QUESTION_PERIOD = const TokenType(
'?.', 'QUESTION_PERIOD', SELECTOR_PRECEDENCE, QUESTION_PERIOD_TOKEN,
isOperator: true);
static const TokenType QUESTION_QUESTION = const TokenType(
'??', 'QUESTION_QUESTION', IF_NULL_PRECEDENCE, QUESTION_QUESTION_TOKEN,
isOperator: true);
static const TokenType QUESTION_QUESTION_EQ = const TokenType('??=',
'QUESTION_QUESTION_EQ', ASSIGNMENT_PRECEDENCE, QUESTION_QUESTION_EQ_TOKEN,
isOperator: true);
static const TokenType SEMICOLON =
const TokenType(';', 'SEMICOLON', NO_PRECEDENCE, SEMICOLON_TOKEN);
static const TokenType SLASH = const TokenType(
'/', 'SLASH', MULTIPLICATIVE_PRECEDENCE, SLASH_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType SLASH_EQ = const TokenType(
'/=', 'SLASH_EQ', ASSIGNMENT_PRECEDENCE, SLASH_EQ_TOKEN,
isOperator: true);
static const TokenType STAR = const TokenType(
'*', 'STAR', MULTIPLICATIVE_PRECEDENCE, STAR_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType STAR_EQ = const TokenType(
'*=', 'STAR_EQ', ASSIGNMENT_PRECEDENCE, STAR_EQ_TOKEN,
isOperator: true);
static const TokenType STRING_INTERPOLATION_EXPRESSION = const TokenType(
'\${',
'STRING_INTERPOLATION_EXPRESSION',
NO_PRECEDENCE,
STRING_INTERPOLATION_TOKEN);
static const TokenType STRING_INTERPOLATION_IDENTIFIER = const TokenType(
'\$',
'STRING_INTERPOLATION_IDENTIFIER',
NO_PRECEDENCE,
STRING_INTERPOLATION_IDENTIFIER_TOKEN);
static const TokenType TILDE = const TokenType(
'~', 'TILDE', PREFIX_PRECEDENCE, TILDE_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType TILDE_SLASH = const TokenType(
'~/', 'TILDE_SLASH', MULTIPLICATIVE_PRECEDENCE, TILDE_SLASH_TOKEN,
isOperator: true, isUserDefinableOperator: true);
static const TokenType TILDE_SLASH_EQ = const TokenType(
'~/=', 'TILDE_SLASH_EQ', ASSIGNMENT_PRECEDENCE, TILDE_SLASH_EQ_TOKEN,
isOperator: true);
static const TokenType BACKPING =
const TokenType('`', 'BACKPING', NO_PRECEDENCE, BACKPING_TOKEN);
static const TokenType BACKSLASH =
const TokenType('\\', 'BACKSLASH', NO_PRECEDENCE, BACKSLASH_TOKEN);
static const TokenType PERIOD_PERIOD_PERIOD = const TokenType(
'...', 'PERIOD_PERIOD_PERIOD', NO_PRECEDENCE, PERIOD_PERIOD_PERIOD_TOKEN);
static const TokenType PERIOD_PERIOD_PERIOD_QUESTION = const TokenType(
'...?',
'PERIOD_PERIOD_PERIOD_QUESTION',
NO_PRECEDENCE,
PERIOD_PERIOD_PERIOD_QUESTION_TOKEN);
static const TokenType QUESTION_PERIOD_OPEN_SQUARE_BRACKET = const TokenType(
'?.[',
'QUESTION_PERIOD_OPEN_SQUARE_BRACKET',
SELECTOR_PRECEDENCE,
QUESTION_PERIOD_OPEN_SQUARE_BRACKET_TOKEN);
static const TokenType QUESTION_PERIOD_PERIOD = const TokenType(
'?..',
'QUESTION_PERIOD_PERIOD',
CASCADE_PRECEDENCE,
QUESTION_PERIOD_PERIOD_TOKEN);
static const TokenType AS = Keyword.AS;
static const TokenType IS = Keyword.IS;
/**
* Token type used by error tokens.
*/
static const TokenType BAD_INPUT = const TokenType(
'malformed input', 'BAD_INPUT', NO_PRECEDENCE, BAD_INPUT_TOKEN,
stringValue: null);
/**
* Token type used by synthetic tokens that are created during parser
* recovery (non-analyzer use case).
*/
static const TokenType RECOVERY = const TokenType(
'recovery', 'RECOVERY', NO_PRECEDENCE, RECOVERY_TOKEN,
stringValue: null);
// TODO(danrubel): "all" is misleading
// because this list does not include all TokenType instances.
static const List<TokenType> all = const <TokenType>[
TokenType.EOF,
TokenType.DOUBLE,
TokenType.HEXADECIMAL,
TokenType.IDENTIFIER,
TokenType.INT,
TokenType.MULTI_LINE_COMMENT,
TokenType.SCRIPT_TAG,
TokenType.SINGLE_LINE_COMMENT,
TokenType.STRING,
TokenType.AMPERSAND,
TokenType.AMPERSAND_AMPERSAND,
TokenType.AMPERSAND_EQ,
TokenType.AT,
TokenType.BANG,
TokenType.BANG_EQ,
TokenType.BAR,
TokenType.BAR_BAR,
TokenType.BAR_EQ,
TokenType.COLON,
TokenType.COMMA,
TokenType.CARET,
TokenType.CARET_EQ,
TokenType.CLOSE_CURLY_BRACKET,
TokenType.CLOSE_PAREN,
TokenType.CLOSE_SQUARE_BRACKET,
TokenType.EQ,
TokenType.EQ_EQ,
TokenType.FUNCTION,
TokenType.GT,
TokenType.GT_EQ,
TokenType.GT_GT,
TokenType.GT_GT_EQ,
TokenType.HASH,
TokenType.INDEX,
TokenType.INDEX_EQ,
TokenType.LT,
TokenType.LT_EQ,
TokenType.LT_LT,
TokenType.LT_LT_EQ,
TokenType.MINUS,
TokenType.MINUS_EQ,
TokenType.MINUS_MINUS,
TokenType.OPEN_CURLY_BRACKET,
TokenType.OPEN_PAREN,
TokenType.OPEN_SQUARE_BRACKET,
TokenType.PERCENT,
TokenType.PERCENT_EQ,
TokenType.PERIOD,
TokenType.PERIOD_PERIOD,
TokenType.PLUS,
TokenType.PLUS_EQ,
TokenType.PLUS_PLUS,
TokenType.QUESTION,
TokenType.QUESTION_PERIOD,
TokenType.QUESTION_QUESTION,
TokenType.QUESTION_QUESTION_EQ,
TokenType.SEMICOLON,
TokenType.SLASH,
TokenType.SLASH_EQ,
TokenType.STAR,
TokenType.STAR_EQ,
TokenType.STRING_INTERPOLATION_EXPRESSION,
TokenType.STRING_INTERPOLATION_IDENTIFIER,
TokenType.TILDE,
TokenType.TILDE_SLASH,
TokenType.TILDE_SLASH_EQ,
TokenType.BACKPING,
TokenType.BACKSLASH,
TokenType.PERIOD_PERIOD_PERIOD,
TokenType.PERIOD_PERIOD_PERIOD_QUESTION,
// TODO(danrubel): Should these be added to the "all" list?
//TokenType.IS,
//TokenType.AS,
// These are not yet part of the language and not supported by fasta
//TokenType.AMPERSAND_AMPERSAND_EQ,
//TokenType.BAR_BAR_EQ,
// Supported by fasta but not part of the language
//TokenType.BANG_EQ_EQ,
//TokenType.EQ_EQ_EQ,
// Used by synthetic tokens generated during recovery
//TokenType.BAD_INPUT,
//TokenType.RECOVERY,
];
final int kind;
/**
* `true` if this token type represents a modifier
* such as `abstract` or `const`.
*/
final bool isModifier;
/**
* `true` if this token type represents an operator.
*/
final bool isOperator;
/**
* `true` if this token type represents a keyword starting a top level
* declaration such as `class`, `enum`, `import`, etc.
*/
final bool isTopLevelKeyword;
/**
* `true` if this token type represents an operator
* that can be defined by users.
*/
final bool isUserDefinableOperator;
/**
* The lexeme that defines this type of token,
* or `null` if there is more than one possible lexeme for this type of token.
*/
final String lexeme;
/**
* The name of the token type.
*/
final String name;
/**
* The precedence of this type of token,
* or `0` if the token does not represent an operator.
*/
final int precedence;
/**
* See [Token.stringValue] for an explanation.
*/
final String stringValue;
const TokenType(this.lexeme, this.name, this.precedence, this.kind,
{this.isModifier: false,
this.isOperator: false,
this.isTopLevelKeyword: false,
this.isUserDefinableOperator: false,
String stringValue: 'unspecified'})
: this.stringValue = stringValue == 'unspecified' ? lexeme : stringValue;
/**
* Return `true` if this type of token represents an additive operator.
*/
bool get isAdditiveOperator => precedence == ADDITIVE_PRECEDENCE;
/**
* Return `true` if this type of token represents an assignment operator.
*/
bool get isAssignmentOperator => precedence == ASSIGNMENT_PRECEDENCE;
/**
* Return `true` if this type of token represents an associative operator. An
* associative operator is an operator for which the following equality is
* true: `(a * b) * c == a * (b * c)`. In other words, if the result of
* applying the operator to multiple operands does not depend on the order in
* which those applications occur.
*
* Note: This method considers the logical-and and logical-or operators to be
* associative, even though the order in which the application of those
* operators can have an effect because evaluation of the right-hand operand
* is conditional.
*/
bool get isAssociativeOperator =>
this == TokenType.AMPERSAND ||
this == TokenType.AMPERSAND_AMPERSAND ||
this == TokenType.BAR ||
this == TokenType.BAR_BAR ||
this == TokenType.CARET ||
this == TokenType.PLUS ||
this == TokenType.STAR;
/**
* A flag indicating whether the keyword is a "built-in" identifier.
*/
bool get isBuiltIn => false;
/// A flag indicating whether the keyword is a "reserved word".
bool get isReservedWord => false;
/**
* Return `true` if this type of token represents an equality operator.
*/
bool get isEqualityOperator =>
this == TokenType.BANG_EQ || this == TokenType.EQ_EQ;
/**
* Return `true` if this type of token represents an increment operator.
*/
bool get isIncrementOperator =>
this == TokenType.PLUS_PLUS || this == TokenType.MINUS_MINUS;
/**
* Return `true` if this type of token is a keyword.
*/
bool get isKeyword => kind == KEYWORD_TOKEN;
/**
* A flag indicating whether the keyword can be used as an identifier
* in some situations.
*/
bool get isPseudo => false;
/**
* Return `true` if this type of token represents a multiplicative operator.
*/
bool get isMultiplicativeOperator => precedence == MULTIPLICATIVE_PRECEDENCE;
/**
* Return `true` if this type of token represents a relational operator.
*/
bool get isRelationalOperator =>
this == TokenType.LT ||
this == TokenType.LT_EQ ||
this == TokenType.GT ||
this == TokenType.GT_EQ;
/**
* Return `true` if this type of token represents a shift operator.
*/
bool get isShiftOperator => precedence == SHIFT_PRECEDENCE;
/**
* Return `true` if this type of token represents a unary postfix operator.
*/
bool get isUnaryPostfixOperator => precedence == POSTFIX_PRECEDENCE;
/**
* Return `true` if this type of token represents a unary prefix operator.
*/
bool get isUnaryPrefixOperator =>
precedence == PREFIX_PRECEDENCE ||
this == TokenType.MINUS ||
this == TokenType.PLUS_PLUS ||
this == TokenType.MINUS_MINUS;
/**
* Return `true` if this type of token represents a selector operator
* (starting token of a selector).
*/
bool get isSelectorOperator => precedence == SELECTOR_PRECEDENCE;
@override
String toString() => name;
}