| // Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'package:_fe_analyzer_shared/src/scanner/errors.dart' |
| show translateErrorToken; |
| import 'package:_fe_analyzer_shared/src/scanner/scanner.dart' as fasta; |
| import 'package:_fe_analyzer_shared/src/scanner/token.dart' |
| show Token, TokenType; |
| import 'package:analyzer/dart/analysis/features.dart'; |
| import 'package:analyzer/error/error.dart'; |
| import 'package:analyzer/error/listener.dart'; |
| import 'package:analyzer/src/dart/error/syntactic_errors.dart'; |
| import 'package:analyzer/src/dart/scanner/reader.dart'; |
| import 'package:analyzer/src/generated/source.dart'; |
| import 'package:meta/meta.dart'; |
| import 'package:pub_semver/pub_semver.dart'; |
| |
| export 'package:analyzer/src/dart/error/syntactic_errors.dart'; |
| |
| /// The class `Scanner` implements a scanner for Dart code. |
| /// |
| /// The lexical structure of Dart is ambiguous without knowledge of the context |
| /// in which a token is being scanned. For example, without context we cannot |
| /// determine whether source of the form "<<" should be scanned as a single |
| /// left-shift operator or as two left angle brackets. This scanner does not |
| /// have any context, so it always resolves such conflicts by scanning the |
| /// longest possible token. |
| class Scanner { |
| final Source source; |
| |
| /// The text to be scanned. |
| final String _contents; |
| |
| /// The offset of the first character from the reader. |
| final int _readerOffset; |
| |
| /// The error listener that will be informed of any errors that are found |
| /// during the scan. |
| final AnalysisErrorListener _errorListener; |
| |
| /// If the file has [fasta.LanguageVersionToken], it is allowed to use the |
| /// language version greater than the one specified in the package config. |
| /// So, we need to know the full feature set for the context. |
| FeatureSet _featureSetForOverriding; |
| |
| /// The flag specifying whether documentation comments should be parsed. |
| bool _preserveComments = true; |
| |
| final List<int> lineStarts = <int>[]; |
| |
| Token firstToken; |
| |
| /// A flag indicating whether the scanner should recognize the `>>>` operator |
| /// and the `>>>=` operator. |
| /// |
| /// Use [configureFeatures] rather than this field. |
| bool enableGtGtGt = false; |
| |
| /// A flag indicating whether the scanner should recognize the `late` and |
| /// `required` keywords. |
| /// |
| /// Use [configureFeatures] rather than this field. |
| bool enableNonNullable = false; |
| |
| fasta.LanguageVersionToken _languageVersion; |
| |
| FeatureSet _featureSet; |
| |
| /// Initialize a newly created scanner to scan characters from the given |
| /// [source]. The given character [reader] will be used to read the characters |
| /// in the source. The given [_errorListener] will be informed of any errors |
| /// that are found. |
| factory Scanner(Source source, CharacterReader reader, |
| AnalysisErrorListener errorListener) => |
| Scanner.fasta(source, errorListener, |
| contents: reader.getContents(), offset: reader.offset); |
| |
| factory Scanner.fasta(Source source, AnalysisErrorListener errorListener, |
| {String contents, int offset = -1}) { |
| return Scanner._( |
| source, contents ?? source.contents.data, offset, errorListener); |
| } |
| |
| Scanner._( |
| this.source, this._contents, this._readerOffset, this._errorListener) { |
| lineStarts.add(0); |
| } |
| |
| /// The features associated with this scanner. |
| /// |
| /// If a language version comment (e.g. '// @dart = 2.3') is detected |
| /// when calling [tokenize] and this field is non-null, then this field |
| /// will be updated to contain a downgraded feature set based upon the |
| /// language version specified. |
| /// |
| /// Use [configureFeatures] to set the features. |
| FeatureSet get featureSet => _featureSet; |
| |
| /// The language version override specified for this compilation unit using a |
| /// token like '// @dart = 2.7', or `null` if no override is specified. |
| fasta.LanguageVersionToken get languageVersion => _languageVersion; |
| |
| set preserveComments(bool preserveComments) { |
| _preserveComments = preserveComments; |
| } |
| |
| /// Configures the scanner appropriately for the given [featureSet]. |
| /// |
| /// TODO(paulberry): stop exposing `enableGtGtGt` and `enableNonNullable` so |
| /// that callers are forced to use this API. Note that this would be a |
| /// breaking change. |
| void configureFeatures({ |
| @required FeatureSet featureSetForOverriding, |
| @required FeatureSet featureSet, |
| }) { |
| _featureSetForOverriding = featureSetForOverriding; |
| _featureSet = featureSet; |
| enableGtGtGt = featureSet.isEnabled(Feature.triple_shift); |
| enableNonNullable = featureSet.isEnabled(Feature.non_nullable); |
| } |
| |
| void reportError( |
| ScannerErrorCode errorCode, int offset, List<Object> arguments) { |
| _errorListener |
| .onError(AnalysisError(source, offset, 1, errorCode, arguments)); |
| } |
| |
| void setSourceStart(int line, int column) { |
| int offset = _readerOffset; |
| if (line < 1 || column < 1 || offset < 0 || (line + column - 2) >= offset) { |
| return; |
| } |
| lineStarts.removeAt(0); |
| for (int i = 2; i < line; i++) { |
| lineStarts.add(1); |
| } |
| lineStarts.add(offset - column + 1); |
| } |
| |
| /// The fasta parser handles error tokens produced by the scanner |
| /// but the old parser used by angular does not |
| /// and expects that scanner errors to be reported by this method. |
| /// Set [reportScannerErrors] `true` when using the old parser. |
| Token tokenize({bool reportScannerErrors = true}) { |
| fasta.ScannerResult result = fasta.scanString(_contents, |
| configuration: _featureSet != null |
| ? buildConfig(_featureSet) |
| : fasta.ScannerConfiguration( |
| enableTripleShift: enableGtGtGt, |
| enableNonNullable: enableNonNullable), |
| includeComments: _preserveComments, |
| languageVersionChanged: _languageVersionChanged); |
| |
| // fasta pretends there is an additional line at EOF |
| result.lineStarts.removeLast(); |
| |
| // for compatibility, there is already a first entry in lineStarts |
| result.lineStarts.removeAt(0); |
| |
| lineStarts.addAll(result.lineStarts); |
| fasta.Token token = result.tokens; |
| |
| // The fasta parser handles error tokens produced by the scanner |
| // but the old parser used by angular does not |
| // and expects that scanner errors to be reported here |
| if (reportScannerErrors) { |
| // The default recovery strategy used by scanString |
| // places all error tokens at the head of the stream. |
| while (token.type == TokenType.BAD_INPUT) { |
| translateErrorToken(token, reportError); |
| token = token.next; |
| } |
| } |
| |
| firstToken = token; |
| // Update all token offsets based upon the reader's starting offset |
| if (_readerOffset != -1) { |
| final int delta = _readerOffset + 1; |
| do { |
| token.offset += delta; |
| token = token.next; |
| } while (!token.isEof); |
| } |
| return firstToken; |
| } |
| |
| void _languageVersionChanged( |
| fasta.Scanner scanner, fasta.LanguageVersionToken languageVersion) { |
| if (languageVersion.major >= 0 && languageVersion.minor >= 0) { |
| _languageVersion = languageVersion; |
| if (_featureSet != null) { |
| _featureSet = _featureSetForOverriding.restrictToVersion( |
| Version(languageVersion.major, languageVersion.minor, 0), |
| ); |
| scanner.configuration = buildConfig(_featureSet); |
| } |
| } |
| } |
| |
| /// Return a ScannerConfiguration based upon the specified feature set. |
| static fasta.ScannerConfiguration buildConfig(FeatureSet featureSet) => |
| featureSet == null |
| ? fasta.ScannerConfiguration() |
| : fasta.ScannerConfiguration( |
| enableExtensionMethods: |
| featureSet.isEnabled(Feature.extension_methods), |
| enableTripleShift: featureSet.isEnabled(Feature.triple_shift), |
| enableNonNullable: featureSet.isEnabled(Feature.non_nullable)); |
| } |