blob: 44266cae6b4d438e5f403951fa42309513a14662 [file] [log] [blame]
// Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
library dart2js.scanner.string_scanner;
import 'characters.dart' show $EOF;
import 'token.dart'
show
CommentToken,
LanguageVersionToken,
SyntheticStringToken,
Token,
TokenType;
import 'token.dart' as analyzer show StringToken;
import 'abstract_scanner.dart'
show AbstractScanner, LanguageVersionChanged, ScannerConfiguration;
import 'string_canonicalizer.dart'
show canonicalizeString, canonicalizeSubString;
import 'token_impl.dart'
show
CommentTokenImpl,
DartDocToken,
LanguageVersionTokenImpl,
StringTokenImpl;
import 'error_token.dart' show ErrorToken;
/**
* Scanner that reads from a String and creates tokens that points to
* substrings.
*/
class StringScanner extends AbstractScanner {
/** The file content. */
final String _string;
final int _stringLengthMinusOne;
/** The current offset in [_string]. */
@override
int scanOffset = -1;
StringScanner(this._string,
{ScannerConfiguration? configuration,
bool includeComments = false,
LanguageVersionChanged? languageVersionChanged})
: _stringLengthMinusOne = _string.length - 1,
super(configuration, includeComments, languageVersionChanged,
numberOfBytesHint: _string.length);
StringScanner.recoveryOptionScanner(StringScanner super.copyFrom)
: _string = copyFrom._string,
_stringLengthMinusOne = copyFrom._stringLengthMinusOne,
scanOffset = copyFrom.scanOffset,
super.recoveryOptionScanner();
@override
StringScanner createRecoveryOptionScanner() {
return new StringScanner.recoveryOptionScanner(this);
}
static bool isLegalIdentifier(String identifier) {
StringScanner scanner = new StringScanner(identifier);
Token startToken = scanner.tokenize();
return startToken is! ErrorToken && startToken.next!.isEof;
}
@override
@pragma('vm:unsafe:no-bounds-checks')
int advance() {
// Always increment so scanOffset goes past the end.
++scanOffset;
if (scanOffset > _stringLengthMinusOne) return $EOF;
return _string.codeUnitAt(scanOffset);
}
@override
@pragma('vm:unsafe:no-bounds-checks')
int peek() {
int next = scanOffset + 1;
if (next > _stringLengthMinusOne) return $EOF;
return _string.codeUnitAt(next);
}
@override
int get stringOffset => scanOffset;
@override
int currentAsUnicode(int next) => next;
@override
void handleUnicode(int startScanOffset) {}
@override
analyzer.StringToken createSubstringToken(TokenType type, int start,
bool asciiOnly, int extraOffset, bool allowLazy) {
return new StringTokenImpl.fromSubstring(
type, _string, start, scanOffset + extraOffset, tokenStart,
canonicalize: true,
precedingComments: comments,
allowLazyFoo: allowLazy);
}
@override
analyzer.StringToken createSyntheticSubstringToken(
TokenType type, int start, bool asciiOnly, String syntheticChars) {
String value = syntheticChars.length == 0
? canonicalizeSubString(_string, start, scanOffset)
: canonicalizeString(
_string.substring(start, scanOffset) + syntheticChars);
return new SyntheticStringToken(
type, value, tokenStart, value.length - syntheticChars.length);
}
@override
CommentToken createCommentToken(TokenType type, int start, bool asciiOnly,
[int extraOffset = 0]) {
return new CommentTokenImpl.fromSubstring(
type, _string, start, scanOffset + extraOffset, tokenStart,
canonicalize: true);
}
@override
DartDocToken createDartDocToken(TokenType type, int start, bool asciiOnly,
[int extraOffset = 0]) {
return new DartDocToken.fromSubstring(
type, _string, start, scanOffset + extraOffset, tokenStart,
canonicalize: true);
}
@override
LanguageVersionToken createLanguageVersionToken(
int start, int major, int minor) {
return new LanguageVersionTokenImpl.fromSubstring(
_string, start, scanOffset, tokenStart, major, minor,
canonicalize: true);
}
@override
// This class used to enforce zero-terminated input, so we only return true
// once advance has been out of bounds.
// TODO(jensj): This should probably change.
// It's at least used in tests (where the eof token has its offset reduced
// by one to 'fix' this.)
bool atEndOfFile() => scanOffset > _stringLengthMinusOne;
}