Initial stress test for code completion
Change-Id: I5ec7ff846da2924ef26a7e738a5e30191a3df9c5
Reviewed-on: https://dart-review.googlesource.com/68440
Commit-Queue: Brian Wilkerson <brianwilkerson@google.com>
Reviewed-by: Konstantin Shcheglov <scheglov@google.com>
diff --git a/pkg/analysis_server/test/stress/completion/completion.dart b/pkg/analysis_server/test/stress/completion/completion.dart
new file mode 100644
index 0000000..9581943
--- /dev/null
+++ b/pkg/analysis_server/test/stress/completion/completion.dart
@@ -0,0 +1,100 @@
+// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:args/args.dart';
+
+import 'completion_runner.dart';
+
+/**
+ * The main entry point for the code completion stress test.
+ */
+void main(List<String> args) async {
+ ArgParser parser = createArgParser();
+ ArgResults result = parser.parse(args);
+
+ if (validArguments(parser, result)) {
+ String analysisRoot = result.rest[0];
+
+ CompletionRunner runner = new CompletionRunner(
+ output: stdout,
+ printMissing: result['missing'],
+ timing: result['timing'],
+ useCFE: result['use-cfe'],
+ verbose: result['verbose']);
+ await runner.runAll(analysisRoot);
+ stdout.flush();
+ }
+}
+
+/**
+ * Create a parser that can be used to parse the command-line arguments.
+ */
+ArgParser createArgParser() {
+ ArgParser parser = new ArgParser();
+ parser.addFlag(
+ 'help',
+ abbr: 'h',
+ help: 'Print this help message',
+ negatable: false,
+ );
+ parser.addFlag(
+ 'missing',
+ help: 'Report locations where the current identifier was not suggested',
+ negatable: false,
+ );
+ parser.addFlag(
+ 'timing',
+ help: 'Report timing information',
+ negatable: false,
+ );
+ parser.addFlag(
+ 'use-cfe',
+ help: 'Use the CFE to perform the analysis',
+ negatable: false,
+ );
+ parser.addFlag(
+ 'verbose',
+ abbr: 'v',
+ help: 'Produce verbose output',
+ negatable: false,
+ );
+ return parser;
+}
+
+/**
+ * Print usage information for this tool.
+ */
+void printUsage(ArgParser parser, {String error}) {
+ if (error != null) {
+ print(error);
+ print('');
+ }
+ print('usage: dart completion path');
+ print('');
+ print('Test the completion engine by requesting completion at the offset of');
+ print('each identifier in the files contained in the given path. The path');
+ print('can be either a single Dart file or a directory.');
+ print('');
+ print(parser.usage);
+}
+
+/**
+ * Return `true` if the command-line arguments (represented by the [result] and
+ * parsed by the [parser]) are valid.
+ */
+bool validArguments(ArgParser parser, ArgResults result) {
+ if (result.wasParsed('help')) {
+ printUsage(parser);
+ return false;
+ } else if (result.rest.length < 1) {
+ printUsage(parser, error: 'Missing path to files');
+ return false;
+ } else if (result.rest.length > 1) {
+ printUsage(parser, error: 'Only one file can be analyzed');
+ return false;
+ }
+ return true;
+}
diff --git a/pkg/analysis_server/test/stress/completion/completion_runner.dart b/pkg/analysis_server/test/stress/completion/completion_runner.dart
new file mode 100644
index 0000000..2f49a24
--- /dev/null
+++ b/pkg/analysis_server/test/stress/completion/completion_runner.dart
@@ -0,0 +1,224 @@
+// Copyright (c) 2018, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:analysis_server/src/services/completion/completion_core.dart';
+import 'package:analysis_server/src/services/completion/completion_performance.dart';
+import 'package:analysis_server/src/services/completion/dart/completion_manager.dart';
+import 'package:analysis_server/src/utilities/null_string_sink.dart';
+import 'package:analyzer/dart/analysis/analysis_context.dart';
+import 'package:analyzer/dart/analysis/analysis_context_collection.dart';
+import 'package:analyzer/dart/analysis/results.dart';
+import 'package:analyzer/dart/ast/ast.dart';
+import 'package:analyzer/dart/ast/visitor.dart';
+import 'package:analyzer/file_system/overlay_file_system.dart';
+import 'package:analyzer/file_system/physical_file_system.dart';
+import 'package:analyzer/source/line_info.dart';
+import 'package:analyzer/src/generated/source.dart';
+import 'package:analyzer_plugin/protocol/protocol_common.dart';
+
+/**
+ * A runner that can request code completion at the location of each identifier
+ * in a Dart file.
+ */
+class CompletionRunner {
+ /**
+ * The sink to which output is to be written.
+ */
+ final StringSink output;
+
+ /**
+ * A flag indicating whether to produce output about missing suggestions.
+ */
+ final bool printMissing;
+
+ /**
+ * A flag indicating whether to produce timing information.
+ */
+ final bool timing;
+
+ /**
+ * A flag indicating whether to use the CFE when running the tests.
+ */
+ final bool useCFE;
+
+ /**
+ * A flag indicating whether to produce verbose output.
+ */
+ final bool verbose;
+
+ /**
+ * A flag indicating whether we should delete each identifier before
+ * attempting to complete at that offset.
+ */
+ bool deleteBeforeCompletion = false;
+
+ /**
+ * Initialize a newly created completion runner.
+ */
+ CompletionRunner(
+ {StringSink output,
+ bool printMissing,
+ bool timing,
+ bool useCFE,
+ bool verbose})
+ : this.output = output ?? new NullStringSink(),
+ this.printMissing = printMissing ?? false,
+ this.timing = timing ?? false,
+ this.useCFE = useCFE ?? false,
+ this.verbose = verbose ?? false;
+
+ /**
+ * Test the completion engine at the locations of each of the identifiers in
+ * each of the files in the given [analysisRoot].
+ */
+ Future<void> runAll(String analysisRoot) async {
+ OverlayResourceProvider resourceProvider =
+ new OverlayResourceProvider(PhysicalResourceProvider.INSTANCE);
+ AnalysisContextCollection collection = new AnalysisContextCollection(
+ includedPaths: <String>[analysisRoot],
+ resourceProvider: resourceProvider,
+ useCFE: useCFE); // ignore: deprecated_member_use
+ DartCompletionManager contributor = new DartCompletionManager();
+ CompletionPerformance performance = new CompletionPerformance();
+ int stamp = 1;
+
+ int fileCount = 0;
+ int identifierCount = 0;
+ int missingCount = 0;
+
+ // Consider getting individual timings so that we can also report the
+ // longest and shortest times, or even a distribution.
+ Stopwatch timer = new Stopwatch();
+
+ for (AnalysisContext context in collection.contexts) {
+ for (String path in context.contextRoot.analyzedFiles()) {
+ fileCount++;
+ output.write('.');
+ ResolveResult result =
+ await context.currentSession.getResolvedAst(path);
+ String content = result.content;
+ LineInfo lineInfo = result.lineInfo;
+ List<SimpleIdentifier> identifiers = _identifiersIn(result.unit);
+
+ for (SimpleIdentifier identifier in identifiers) {
+ identifierCount++;
+ int offset = identifier.offset;
+ if (deleteBeforeCompletion) {
+ String modifiedContent = content.substring(0, offset) +
+ content.substring(identifier.end);
+ resourceProvider.setOverlay(path,
+ content: modifiedContent, modificationStamp: stamp++);
+ result = await context.currentSession.getResolvedAst(path);
+ }
+
+ timer.start();
+ CompletionRequestImpl request =
+ new CompletionRequestImpl(result, offset, performance);
+ List<CompletionSuggestion> suggestions =
+ await contributor.computeSuggestions(request);
+ timer.stop();
+
+ if (!identifier.inDeclarationContext() &&
+ !_isNamedExpressionName(identifier)) {
+ if (!_hasSuggestion(suggestions, identifier.name)) {
+ missingCount++;
+ if (printMissing) {
+ CharacterLocation location = lineInfo.getLocation(offset);
+ output.writeln('Missing suggestion of "${identifier.name}" at '
+ '$path:${location.lineNumber}:${location.columnNumber}');
+ if (verbose) {
+ _printSuggestions(suggestions);
+ }
+ }
+ }
+ }
+ }
+ if (deleteBeforeCompletion) {
+ resourceProvider.removeOverlay(path);
+ }
+ }
+ }
+ output.writeln();
+ if (printMissing) {
+ output.writeln();
+ }
+ if (identifierCount == 0) {
+ output.writeln('No identifiers found in $fileCount files');
+ } else {
+ int percent = (missingCount * 100 / identifierCount).round();
+ output.writeln('$percent% missing suggestions '
+ '($missingCount of $identifierCount in $fileCount files)');
+ }
+ if (timing && identifierCount > 0) {
+ int time = timer.elapsedMilliseconds;
+ int averageTime = (time / identifierCount).round();
+ output.writeln('completion took $time ms, '
+ 'which is an average of $averageTime ms per completion');
+ }
+ }
+
+ /**
+ * Return `true` if the given list of [suggestions] includes a suggestion for
+ * the given [identifier].
+ */
+ bool _hasSuggestion(
+ List<CompletionSuggestion> suggestions, String identifier) {
+ for (CompletionSuggestion suggestion in suggestions) {
+ if (suggestion.completion == identifier) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /**
+ * Return a list containing information about the identifiers in the given
+ * compilation [unit].
+ */
+ List<SimpleIdentifier> _identifiersIn(CompilationUnit unit) {
+ IdentifierCollector visitor = new IdentifierCollector();
+ unit.accept(visitor);
+ return visitor.identifiers;
+ }
+
+ /**
+ * Return `true` if the given [identifier] is being used as the name of a
+ * named expression.
+ */
+ bool _isNamedExpressionName(SimpleIdentifier identifier) {
+ AstNode parent = identifier.parent;
+ return parent is NamedExpression && parent.name == identifier;
+ }
+
+ /**
+ * Print information about the given [suggestions].
+ */
+ void _printSuggestions(List<CompletionSuggestion> suggestions) {
+ if (suggestions.length == 0) {
+ output.writeln(' No suggestions');
+ return;
+ }
+ output.writeln(' Suggestions:');
+ for (CompletionSuggestion suggestion in suggestions) {
+ output.writeln(' ${suggestion.completion}');
+ }
+ }
+}
+
+/**
+ * A visitor that will collect simple identifiers in the AST being visited.
+ */
+class IdentifierCollector extends RecursiveAstVisitor<void> {
+ /**
+ * The simple identifiers that were collected.
+ */
+ final List<SimpleIdentifier> identifiers = <SimpleIdentifier>[];
+
+ @override
+ visitSimpleIdentifier(SimpleIdentifier node) {
+ identifiers.add(node);
+ }
+}
