pkg/analyzer/lib/src/dart/scanner/scanner.dart

// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:typed_data';

import 'package:_fe_analyzer_shared/src/base/errors.dart';
import 'package:_fe_analyzer_shared/src/scanner/scanner.dart' as fasta;
import 'package:_fe_analyzer_shared/src/scanner/token.dart'
    show Token, TokenType;
import 'package:analyzer/dart/analysis/features.dart';
import 'package:analyzer/source/source.dart';
import 'package:analyzer/src/dart/analysis/experiments.dart';
import 'package:analyzer/src/dart/scanner/translate_error_token.dart'
    show translateErrorToken;
import 'package:analyzer/src/diagnostic/diagnostic.dart' as diag;
import 'package:analyzer/src/error/listener.dart';
import 'package:analyzer/src/utilities/extensions/source.dart';
import 'package:meta/meta.dart';
import 'package:pub_semver/pub_semver.dart';

/// The class `Scanner` implements a scanner for Dart code.
///
/// The lexical structure of Dart is ambiguous without knowledge of the context
/// in which a token is being scanned. For example, without context we cannot
/// determine whether source of the form "<<" should be scanned as a single
/// left-shift operator or as two left angle brackets. This scanner does not
/// have any context, so it always resolves such conflicts by scanning the
/// longest possible token.
class Scanner {
  static final Uint8List _lineStartsZero = Uint8List(0);

  /// Allows to - by default - either preserve or not preserve comments while
  /// testing.
  @visibleForTesting
  static bool preserveCommentsDefaultForTesting = true;

  final Source source;

  /// The text to be scanned.
  final String _contents;

  /// The offset of the first character from the reader.
  final int _readerOffset;

  /// The diagnostic listener that will be informed of any diagnostics that are
  /// found during the scan.
  final DiagnosticReporter _diagnosticReporter;

  /// If the file has [fasta.LanguageVersionToken], it is allowed to use the
  /// language version greater than the one specified in the package config.
  /// So, we need to know the full feature set for the context.
  late final FeatureSet _featureSetForOverriding;

  /// The flag specifying whether documentation comments should be parsed.
  bool _preserveComments = preserveCommentsDefaultForTesting;
  List<int>? _lineStarts;
  late final Token firstToken;

  Version? _overrideVersion;

  late FeatureSet _featureSet;

  /// Initializes a scanner to scan the given [contents].
  ///
  /// The [diagnosticReporter] will be informed of any errors that are found.
  factory Scanner(String contents, DiagnosticReporter diagnosticReporter) =>
      Scanner.fasta(diagnosticReporter, contents: contents);

  factory Scanner.fasta(
    DiagnosticReporter diagnosticReporter, {
    String? contents,
    int offset = -1,
  }) {
    return Scanner._(
      diagnosticReporter.source,
      contents ?? diagnosticReporter.source.stringContents,
      offset,
      diagnosticReporter,
    );
  }

  Scanner._(
    this.source,
    this._contents,
    this._readerOffset,
    this._diagnosticReporter,
  );

  /// The features associated with this scanner.
  ///
  /// If a language version comment (e.g. '// @dart = 2.3') is detected
  /// when calling [tokenize] and this field is non-null, then this field
  /// will be updated to contain a downgraded feature set based upon the
  /// language version specified.
  ///
  /// Use [configureFeatures] to set the features.
  FeatureSet get featureSet => _featureSet;

  List<int> get lineStarts => _lineStarts ?? _lineStartsZero;

  /// The language version override specified for this compilation unit using a
  /// token like '// @dart = 2.7', or `null` if no override is specified.
  Version? get overrideVersion => _overrideVersion;

  set preserveComments(bool preserveComments) {
    _preserveComments = preserveComments;
  }

  /// Configures the scanner appropriately for the given [featureSet].
  void configureFeatures({
    required FeatureSet featureSetForOverriding,
    required FeatureSet featureSet,
  }) {
    _featureSetForOverriding = featureSetForOverriding;
    _featureSet = featureSet;
  }

  void reportError(LocatedDiagnostic locatedDiagnostic) {
    _diagnosticReporter.report(locatedDiagnostic);
  }

  /// The fasta parser handles error tokens produced by the scanner
  /// but the old parser used by angular does not
  /// and expects that scanner errors to be reported by this method.
  /// Set [reportScannerErrors] `true` when using the old parser.
  Token tokenize({bool reportScannerErrors = true}) {
    fasta.ScannerResult result = fasta.scanString(
      _contents,
      configuration: buildConfig(_featureSet),
      includeComments: _preserveComments,
      languageVersionChanged: _languageVersionChanged,
    );

    // fasta pretends there is an additional line at EOF so we skip the last one.
    if (result.lineStarts.last > 65535) {
      Uint32List list = _lineStarts = Uint32List(result.lineStarts.length - 1);
      list.setRange(0, result.lineStarts.length - 1, result.lineStarts);
    } else {
      Uint16List list = _lineStarts = Uint16List(result.lineStarts.length - 1);
      list.setRange(0, result.lineStarts.length - 1, result.lineStarts);
    }

    fasta.Token token = result.tokens;

    // The fasta parser handles error tokens produced by the scanner
    // but the old parser used by angular does not
    // and expects that scanner errors to be reported here
    if (reportScannerErrors) {
      // The default recovery strategy used by scanString
      // places all error tokens at the head of the stream.
      while (token.type == TokenType.BAD_INPUT) {
        translateErrorToken(token as fasta.ErrorToken, reportError);
        token = token.next!;
      }
    }

    firstToken = token;
    // Update all token offsets based upon the reader's starting offset
    if (_readerOffset != -1) {
      int delta = _readerOffset + 1;
      do {
        token.offset += delta;
        token = token.next!;
      } while (!token.isEof);
    }
    return firstToken;
  }

  void _languageVersionChanged(
    fasta.Scanner scanner,
    fasta.LanguageVersionToken versionToken,
  ) {
    var overrideMajor = versionToken.major;
    var overrideMinor = versionToken.minor;
    if (overrideMajor < 0 || overrideMinor < 0) {
      return;
    }

    var overrideVersion = Version(overrideMajor, overrideMinor, 0);
    _overrideVersion = overrideVersion;

    var latestVersion = ExperimentStatus.currentVersion;
    if (overrideVersion > latestVersion) {
      _diagnosticReporter.report(
        diag.invalidLanguageVersionOverrideGreater
            .withArguments(
              latestMajor: latestVersion.major,
              latestMinor: latestVersion.minor,
            )
            .atOffset(offset: versionToken.offset, length: versionToken.length),
      );
      _overrideVersion = null;
    } else {
      _featureSet = _featureSetForOverriding.restrictToVersion(overrideVersion);
      scanner.configuration = buildConfig(_featureSet);
    }
  }

  /// Return a ScannerConfiguration based upon the specified feature set.
  static fasta.ScannerConfiguration buildConfig(FeatureSet? featureSet) =>
      featureSet == null
      ? fasta.ScannerConfiguration()
      : fasta.ScannerConfiguration(
          enableTripleShift: featureSet.isEnabled(Feature.triple_shift),
          forAugmentationLibrary: featureSet.isEnabled(Feature.macros),
        );
}