pkg/analysis_server/test/stress/completion/completion_runner.dart

// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'package:analysis_server/src/services/completion/dart/completion_manager.dart';
import 'package:analysis_server/src/services/completion/dart/suggestion_builder.dart';
import 'package:analysis_server/src/utilities/null_string_sink.dart';
import 'package:analyzer/dart/analysis/analysis_context_collection.dart';
import 'package:analyzer/dart/analysis/results.dart';
import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/dart/ast/visitor.dart';
import 'package:analyzer/file_system/overlay_file_system.dart';
import 'package:analyzer/file_system/physical_file_system.dart';
import 'package:analyzer/src/util/performance/operation_performance.dart';

/// A runner that can request code completion at the location of each identifier
/// in a Dart file.
class CompletionRunner {
  /// The sink to which output is to be written.
  final StringSink output;

  /// A flag indicating whether to produce output about missing suggestions.
  final bool printMissing;

  /// A flag indicating whether to produce output about the quality of the sort
  /// order.
  final bool printQuality;

  /// A flag indicating whether to produce timing information.
  final bool timing;

  /// A flag indicating whether to produce verbose output.
  final bool verbose;

  /// A flag indicating whether we should delete each identifier before
  /// attempting to complete at that offset.
  bool deleteBeforeCompletion = false;

  /// Initialize a newly created completion runner.
  new({
    StringSink? output,
    bool? printMissing,
    bool? printQuality,
    bool? timing,
    bool? verbose,
  }) : output = output ?? NullStringSink(),
       printMissing = printMissing ?? false,
       printQuality = printQuality ?? false,
       timing = timing ?? false,
       verbose = verbose ?? false;

  /// Test the completion engine at the locations of each of the identifiers in
  /// each of the files in the given [analysisRoot].
  Future<void> runAll(String analysisRoot) async {
    var resourceProvider = OverlayResourceProvider(
      PhysicalResourceProvider.INSTANCE,
    );
    var collection = AnalysisContextCollection(
      includedPaths: <String>[analysisRoot],
      resourceProvider: resourceProvider,
    );
    var contributor = DartCompletionManager(
      budget: CompletionBudget(const Duration(seconds: 30)),
    );
    var stamp = 1;

    var fileCount = 0;
    var identifierCount = 0;
    var expectedCount = 0;
    var missingCount = 0;
    var indexCount = List<int>.filled(20, 0);
    var filteredIndexCount = List<int>.filled(20, 0);

    // Consider getting individual timings so that we can also report the
    // longest and shortest times, or even a distribution.
    var timer = Stopwatch();

    for (var context in collection.contexts) {
      for (var path in context.contextRoot.analyzedFiles()) {
        if (!path.endsWith('.dart')) {
          continue;
        }
        fileCount++;
        output.write('.');
        var result = await context.currentSession.getResolvedUnit(
          path,
        ) as ResolvedUnitResult;
        var content = result.content;
        var lineInfo = result.lineInfo;
        var identifiers = _identifiersIn(result.unit);

        for (var identifier in identifiers) {
          identifierCount++;
          var offset = identifier.offset;
          if (deleteBeforeCompletion) {
            var modifiedContent =
                content.substring(0, offset) +
                content.substring(identifier.end);
            resourceProvider.setOverlay(
              path,
              content: modifiedContent,
              modificationStamp: stamp++,
            );
            result = await context.currentSession.getResolvedUnit(
              path,
            ) as ResolvedUnitResult;
          }

          timer.start();
          var dartRequest = DartCompletionRequest.forResolvedUnit(
            resolvedUnit: result,
            offset: offset,
          );
          var suggestions = await contributor.computeSuggestions(
            dartRequest,
            OperationPerformanceImpl('<root>'),
            maxSuggestions: -1,
            useFilter: true,
          );
          timer.stop();

          if (!identifier.inDeclarationContext()) {
            expectedCount++;
            suggestions = _sort(suggestions.toList());
            var index = _indexOf(suggestions, identifier.name);
            if (index < 0) {
              missingCount++;
              if (printMissing) {
                var location = lineInfo.getLocation(offset);
                output.writeln(
                  'Missing suggestion of "${identifier.name}" at '
                  '$path:${location.lineNumber}:${location.columnNumber}',
                );
                if (verbose) {
                  _printSuggestions(suggestions);
                }
              }
            } else if (printQuality) {
              if (index < indexCount.length) {
                indexCount[index]++;
              }
              var filteredSuggestions = _filterBy(
                suggestions,
                identifier.name.substring(0, 1),
              );
              var filteredIndex = _indexOf(
                filteredSuggestions,
                identifier.name,
              );
              if (filteredIndex < filteredIndexCount.length) {
                filteredIndexCount[filteredIndex]++;
              }
            }
          }
        }
        if (deleteBeforeCompletion) {
          resourceProvider.removeOverlay(path);
        }
      }
    }
    output.writeln();
    if (printMissing) {
      output.writeln();
    }
    output.writeln('Found $identifierCount identifiers in $fileCount files');
    if (expectedCount > 0) {
      output.writeln('  $expectedCount were expected to code complete');
      if (printQuality) {
        var percent = (missingCount * 100 / expectedCount).round();
        output.writeln(
          '  $percent% of which were missing suggestions '
          '($missingCount)',
        );

        var foundCount = expectedCount - missingCount;

        void printCount(int count) {
          if (count < 10) {
            output.write('      $count  ');
          } else if (count < 100) {
            output.write('     $count  ');
          } else if (count < 1000) {
            output.write('    $count  ');
          } else if (count < 10000) {
            output.write('   $count  ');
          } else {
            output.write('  $count  ');
          }
          var percent = (count * 100 / foundCount).floor();
          for (var j = 0; j < percent; j++) {
            output.write('-');
          }
          output.writeln();
        }

        void printCounts(List<int> counts) {
          var nearTopCount = 0;
          for (var i = 0; i < counts.length; i++) {
            var count = counts[i];
            printCount(count);
            nearTopCount += count;
          }
          printCount(foundCount - nearTopCount);
        }

        output.writeln();
        output.writeln('By position in the list');
        printCounts(indexCount);
        output.writeln();
        output.writeln('By position in the list (filtered by first character)');
        printCounts(filteredIndexCount);
        output.writeln();
      }
    }
    if (timing && identifierCount > 0) {
      var time = timer.elapsedMilliseconds;
      var averageTime = (time / identifierCount).round();
      output.writeln(
        'completion took $time ms, '
        'which is an average of $averageTime ms per completion',
      );
    }
  }

  List<CompletionSuggestionBuilder> _filterBy(
    List<CompletionSuggestionBuilder> suggestions,
    String pattern,
  ) {
    return suggestions
        .where((suggestion) => suggestion.completion.startsWith(pattern))
        .toList();
  }

  /// Return a list containing information about the identifiers in the given
  /// compilation [unit].
  List<SimpleIdentifier> _identifiersIn(CompilationUnit unit) {
    var visitor = IdentifierCollector();
    unit.accept(visitor);
    return visitor.identifiers;
  }

  /// If the given list of [suggestions] includes a suggestion for the given
  /// [identifier], return the index of the suggestion. Otherwise, return `-1`.
  int _indexOf(
    List<CompletionSuggestionBuilder> suggestions,
    String identifier,
  ) {
    for (var i = 0; i < suggestions.length; i++) {
      if (suggestions[i].completion == identifier) {
        return i;
      }
    }
    return -1;
  }

  /// Print information about the given [suggestions].
  void _printSuggestions(List<CompletionSuggestionBuilder> suggestions) {
    if (suggestions.isEmpty) {
      output.writeln('  No suggestions');
      return;
    }
    output.writeln('  Suggestions:');
    for (var suggestion in suggestions) {
      output.writeln('    ${suggestion.completion}');
    }
  }

  List<CompletionSuggestionBuilder> _sort(
    List<CompletionSuggestionBuilder> suggestions,
  ) {
    suggestions.sort((first, second) => second.relevance - first.relevance);
    return suggestions;
  }
}

/// A visitor that will collect simple identifiers in the AST being visited.
class IdentifierCollector extends RecursiveAstVisitor<void> {
  /// The simple identifiers that were collected.
  final List<SimpleIdentifier> identifiers = <SimpleIdentifier>[];

  @override
  void visitSimpleIdentifier(SimpleIdentifier node) {
    identifiers.add(node);
  }
}