pkg/scrape/lib/scrape.dart

// Copyright (c) 2020, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

// ignore_for_file: implementation_imports

import 'dart:io';
import 'dart:math' as math;

import 'package:analyzer/dart/analysis/features.dart';
import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/dart/element/element.dart';
import 'package:analyzer/source/line_info.dart';
import 'package:analyzer/src/dart/analysis/experiments.dart';
import 'package:analyzer/src/dart/scanner/scanner.dart';
import 'package:analyzer/src/error/listener.dart';
import 'package:analyzer/src/generated/parser.dart';
import 'package:analyzer/src/string_source.dart';
import 'package:args/args.dart';
import 'package:path/path.dart' as p;

import 'src/error_listener.dart';
import 'src/histogram.dart';
import 'src/scrape_visitor.dart';

export 'src/histogram.dart';
export 'src/scrape_visitor.dart' hide bindVisitor;

class Scrape {
  final List<ScrapeVisitor Function()> _visitorFactories = [];

  /// What percent of files should be processed.
  int? _percent;

  /// Process package test files.
  bool _includeTests = true;

  /// Process Dart SDK language tests.
  bool _includeLanguageTests = true;

  /// Process Dart files generated from protobufs.
  bool _includeProtobufs = false;

  /// Whether every file should be printed before being processed.
  bool _printFiles = true;

  /// Whether parse errors should be printed.
  bool _printErrors = true;

  /// The number of files that have been processed.
  int get scrapedFileCount => _scrapedFileCount;
  int _scrapedFileCount = 0;

  /// The number of lines of code that have been processed.
  int get scrapedLineCount => _scrapedLineCount;
  int _scrapedLineCount = 0;

  /// The number of files that could not be parsed.
  int get errorFileCount => _errorFileCount;
  int _errorFileCount = 0;

  final Map<String, Histogram> _histograms = {};

  /// Whether we're in the middle of writing the running file count and need a
  /// newline before any other output should be shown.
  bool _needClearLine = false;

  /// Register a new visitor factory.
  ///
  /// This function will be called for each scraped file and the resulting
  /// [ScrapeVisitor] will traverse the parsed file's AST.
  void addVisitor(ScrapeVisitor Function() createVisitor) {
    _visitorFactories.add(createVisitor);
  }

  /// Defines a new histogram with [name] to collect occurrences.
  ///
  /// After the scrape completes, each defined histogram's collected counts
  /// are shown, ordered by [order]. If [showBar] is not `false`, then shows an
  /// ASCII bar chart for the counts.
  ///
  /// If [showAll] is `true`, then every item that occurred is shown. Otherwise,
  /// only shows the first 100 items or occurrences that represent at least
  /// 0.1% of the total, whichever is longer.
  ///
  /// If [minCount] is passed, then only shows items that occurred at least
  /// that many times.
  void addHistogram(
    String name, {
    SortOrder order = SortOrder.descending,
    bool? showBar,
    bool? showAll,
    int? minCount,
  }) {
    _histograms.putIfAbsent(
      name,
      () => Histogram(
        order: order,
        showBar: showBar,
        showAll: showAll,
        minCount: minCount,
      ),
    );
  }

  /// Add an occurrence of [item] to [histogram].
  ///
  /// Creates the histogram as needed if not already defined.
  void record(String histogram, Object item) {
    _histograms
        .putIfAbsent(histogram, () => Histogram(order: SortOrder.descending))
        .add(item);
  }

  /// Run the scrape using the given set of command line arguments.
  void runCommandLine(List<String> arguments) {
    var parser = ArgParser(allowTrailingOptions: true);
    parser.addOption(
      'percent',
      help: 'Only process a randomly selected percentage of files.',
    );
    parser.addFlag(
      'tests',
      defaultsTo: true,
      help: 'Process package test files.',
    );
    parser.addFlag(
      'language-tests',
      help: 'Process Dart SDK language test files.',
    );
    parser.addFlag(
      'protobufs',
      help: 'Process Dart files generated from protobufs.',
    );
    parser.addFlag(
      'print-files',
      defaultsTo: true,
      help: 'Print the path for each parsed file.',
    );
    parser.addFlag(
      'print-errors',
      defaultsTo: true,
      help: 'Print parse errors.',
    );
    parser.addFlag('help', negatable: false, help: 'Print help text.');

    var results = parser.parse(arguments);

    if (results['help'] as bool) {
      var script = p.url.basename(Platform.script.toString());
      print('Usage: $script [options] <paths...>');
      print(parser.usage);
      return;
    }

    _includeTests = results['tests'] as bool;
    _includeLanguageTests = results['language-tests'] as bool;
    _includeProtobufs = results['protobufs'] as bool;
    _printFiles = results['print-files'] as bool;
    _printErrors = results['print-errors'] as bool;

    if (results.wasParsed('percent')) {
      _percent = int.tryParse(results['percent'] as String);
      if (_percent == null) {
        print("--percent must be an integer, was '${results["percent"]}'.");
        exit(1);
      }
    }

    if (results.rest.isEmpty) {
      print('Must pass at least one path to process.');
      exit(1);
    }

    var watch = Stopwatch()..start();
    for (var path in results.rest) {
      _processPath(path);
    }
    watch.stop();

    clearLine();
    _histograms.forEach((name, histogram) {
      histogram.printCounts(name);
    });

    String count(int n, String unit) {
      if (n == 1) return '1 $unit';
      return '$n ${unit}s';
    }

    var elapsed = _formatDuration(watch.elapsed);
    var lines = count(_scrapedLineCount, 'line');
    var files = count(_scrapedFileCount, 'file');
    var message = 'Took $elapsed to scrape $lines in $files.';

    if (_errorFileCount > 0) {
      var errors = count(_errorFileCount, 'file');
      message += ' ($errors could not be parsed.)';
    }

    print(message);
  }

  /// Display [message], clearing the line if necessary.
  void log(Object message) {
    // TODO(rnystrom): Consider using cli_util package.
    clearLine();
    print(message);
  }

  /// Clear the current line if it needs it.
  void clearLine() {
    if (!_needClearLine) return;
    stdout.write('\u001b[2K\r');
    _needClearLine = false;
  }

  String _formatDuration(Duration duration) {
    String pad(int width, int n) => n.toString().padLeft(width, '0');

    if (duration.inMinutes >= 1) {
      var minutes = duration.inMinutes;
      var seconds = duration.inSeconds % 60;
      var ms = duration.inMilliseconds % 1000;
      return '$minutes:${pad(2, seconds)}.${pad(3, ms)}';
    } else if (duration.inSeconds >= 1) {
      return '${(duration.inMilliseconds / 1000).toStringAsFixed(3)}s';
    } else {
      return '${duration.inMilliseconds}ms';
    }
  }

  void _processPath(String path) {
    var random = math.Random();

    if (File(path).existsSync()) {
      _parseFile(File(path), path);
      return;
    }

    for (var entry in Directory(path).listSync(recursive: true)) {
      if (entry is! File) continue;

      if (!entry.path.endsWith('.dart')) continue;

      // For unknown reasons, some READMEs have a ".dart" extension. They aren't
      // Dart files.
      if (entry.path.endsWith('README.dart')) continue;

      if (!_includeLanguageTests) {
        if (entry.path.contains('/_fe_analyzer_shared/test/')) continue;
        if (entry.path.contains('/analysis_server/test/')) continue;
        if (entry.path.contains('/analyzer/test/')) continue;
        if (entry.path.contains('/analyzer_cli/test/')) continue;
        if (entry.path.contains('/compiler/test/')) continue;
        if (entry.path.contains('/dart/runtime/observatory/tests/')) continue;
        if (entry.path.contains('/dart/runtime/tests/')) continue;
        if (entry.path.contains('/dart/tests/')) continue;
        if (entry.path.contains('/dev_compiler/test/')) continue;
        if (entry.path.contains('/front_end/parser_testcases/')) continue;
        if (entry.path.contains('/front_end/test/')) continue;
        if (entry.path.contains('/kernel/test/')) continue;
        if (entry.path.contains('/linter/test/_data/')) continue;
        if (entry.path.contains('/testcases/')) continue;
      }

      if (!_includeTests) {
        if (entry.path.contains('/test/')) continue;
        if (entry.path.endsWith('_test.dart')) continue;
      }

      // Don't care about cached packages.
      if (entry.path.contains('sdk/third_party/pkg/')) continue;
      if (entry.path.contains('/.dart_tool/')) continue;

      // Don't care about generated protobuf code.
      if (!_includeProtobufs) {
        if (entry.path.endsWith('.pb.dart')) continue;
        if (entry.path.endsWith('.pbenum.dart')) continue;
      }

      if (_percent != null && random.nextInt(100) >= _percent!) continue;

      var relative = p.relative(entry.path, from: path);
      _parseFile(entry, relative);
    }
  }

  void _parseFile(File file, String shortPath) {
    String source;
    try {
      source = file.readAsStringSync();
    } catch (error) {
      print('Failed to read $shortPath:\n$error');
      return;
    }

    var diagnosticListener = SimpleDiagnosticListener(this, _printErrors);
    var featureSet = FeatureSet.latestLanguageVersion();

    // Tokenize the source.
    var stringSource = StringSource(source, file.path);
    var diagnosticReporter = DiagnosticReporter(
      diagnosticListener,
      stringSource,
    );
    var scanner = Scanner(
      inputText: source,
      reportError: diagnosticReporter.report,
    );
    scanner.configureFeatures(
      featureSet: featureSet,
      featureSetForOverriding: featureSet,
    );
    var startToken = scanner.tokenize();
    var lineInfo = LineInfo(scanner.lineStarts);

    // Parse it.
    var parser = Parser(
      diagnosticReporter,
      featureSet: featureSet,
      languageVersion: LibraryLanguageVersion(
        package: ExperimentStatus.currentVersion,
        override: null,
      ),
      lineInfo: lineInfo,
    );

    if (_printFiles) {
      if (Platform.isWindows) {
        // No ANSI escape codes on Windows.
        var line =
            '[$_scrapedFileCount files, $_scrapedLineCount lines] '
            '$shortPath';
        print(line);
      } else {
        // Overwrite the same line.
        var trimmed = shortPath;
        if (trimmed.length > 80) trimmed = trimmed.substring(0, 80);

        stdout.write(
          '\u001b[2K\r'
          '[$_scrapedFileCount files, $_scrapedLineCount lines] $trimmed',
        );
        _needClearLine = true;
      }
    }

    AstNode node;
    try {
      node = parser.parseCompilationUnit(startToken);
    } catch (error) {
      print('Got exception parsing $shortPath:\n$error');
      return;
    }

    // Don't process files with syntax errors.
    if (diagnosticListener.hadDiagnostic) {
      _errorFileCount++;
      return;
    }

    _scrapedFileCount++;
    _scrapedLineCount += lineInfo.lineCount;

    for (var visitorFactory in _visitorFactories) {
      var visitor = visitorFactory();
      bindVisitor(visitor, this, shortPath, source, startToken, lineInfo);
      node.accept(visitor);
    }
  }
}