Version 2.18.0-84.0.dev
Merge commit '699ea58a4278192144f9293c091407eaae05cd18' into 'dev'
diff --git a/DEPS b/DEPS
index 8501f0d..0ea6033 100644
--- a/DEPS
+++ b/DEPS
@@ -59,7 +59,7 @@
# Checkout extra javascript engines for testing or benchmarking.
# d8, the V8 shell, is always checked out.
"checkout_javascript_engines": False,
- "d8_tag": "version:10.0.40",
+ "d8_tag": "version:10.2.78",
"jsshell_tag": "version:95.0",
# As Flutter does, we use Fuchsia's GN and Clang toolchain. These revision
@@ -85,7 +85,7 @@
"boringssl_rev": "87f316d7748268eb56f2dc147bd593254ae93198",
"browser-compat-data_tag": "ac8cae697014da1ff7124fba33b0b4245cc6cd1b", # v1.0.22
"browser_launcher_rev": "c6cc1025d6901926cf022e144ba109677e3548f1",
- "characters_rev": "6ec389c4dfa8fce14820dc5cbf6e693202e7e052",
+ "characters_rev": "4b1d4b7737ad47cd2b8105c47e2159174010f29f",
"charcode_rev": "84ea427711e24abf3b832923959caa7dd9a8514b",
"chrome_rev": "19997",
"cli_util_rev": "b0adbba89442b2ea6fef39c7a82fe79cb31e1168",
@@ -138,7 +138,7 @@
"pool_rev": "7abe634002a1ba8a0928eded086062f1307ccfae",
"process_rev": "56ece43b53b64c63ae51ec184b76bd5360c28d0b",
"protobuf_rev": "c1eb6cb51af39ccbaa1a8e19349546586a5c8e31",
- "pub_rev": "a949b329b1b51f5f3973a790e0a0a45897d837de",
+ "pub_rev": "6068f47c264ef790e16411b31b2c94ad6beb1ab6",
"pub_semver_rev": "ea6c54019948dc03042c595ce9413e17aaf7aa38",
"root_certificates_rev": "692f6d6488af68e0121317a9c2c9eb393eb0ee50",
"rust_revision": "b7856f695d65a8ebc846754f97d15814bcb1c244",
diff --git a/pkg/analysis_server/tool/code_completion/completion_metrics.dart b/pkg/analysis_server/tool/code_completion/completion_metrics.dart
index ade5fdc..9f14280 100644
--- a/pkg/analysis_server/tool/code_completion/completion_metrics.dart
+++ b/pkg/analysis_server/tool/code_completion/completion_metrics.dart
@@ -19,7 +19,7 @@
import 'package:analysis_server/src/services/completion/dart/suggestion_builder.dart';
import 'package:analysis_server/src/services/completion/dart/utilities.dart';
import 'package:analysis_server/src/status/pages.dart';
-import 'package:analyzer/dart/analysis/context_root.dart';
+import 'package:analyzer/dart/analysis/analysis_context.dart';
import 'package:analyzer/dart/analysis/results.dart';
import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/dart/ast/token.dart';
@@ -38,22 +38,16 @@
PrefixElement,
TypeParameterElement,
VariableElement;
-import 'package:analyzer/diagnostic/diagnostic.dart';
-import 'package:analyzer/error/error.dart' as err;
import 'package:analyzer/file_system/file_system.dart';
-import 'package:analyzer/file_system/overlay_file_system.dart';
import 'package:analyzer/file_system/physical_file_system.dart';
-import 'package:analyzer/src/dart/analysis/analysis_context_collection.dart';
import 'package:analyzer/src/dart/analysis/byte_store.dart';
-import 'package:analyzer/src/dartdoc/dartdoc_directive_info.dart';
import 'package:analyzer/src/services/available_declarations.dart';
-import 'package:analyzer/src/util/file_paths.dart' as file_paths;
import 'package:analyzer/src/util/performance/operation_performance.dart';
import 'package:analyzer_plugin/src/utilities/completion/optype.dart';
import 'package:args/args.dart';
-import 'package:cli_util/cli_logging.dart';
import 'package:collection/collection.dart';
+import 'completion_metrics_base.dart';
import 'metrics_util.dart';
import 'output_utilities.dart';
import 'relevance_table_generator.dart';
@@ -96,12 +90,12 @@
return;
}
- var options = CompletionMetricsOptions(result);
+ var options = CompletionMetricsQualityOptions(result);
var provider = PhysicalResourceProvider.INSTANCE;
if (result.wasParsed('reduceDir')) {
var targetMetrics = <CompletionMetrics>[];
var dir = provider.getFolder(result['reduceDir'] as String);
- var computer = CompletionMetricsComputer('', options);
+ var computer = CompletionQualityMetricsComputer('', options);
for (var child in dir.getChildren()) {
if (child is File) {
var metricsList =
@@ -128,7 +122,7 @@
var rootPath = result.rest[0];
print('Analyzing root: "$rootPath"');
var stopwatch = Stopwatch()..start();
- var computer = CompletionMetricsComputer(rootPath, options);
+ var computer = CompletionQualityMetricsComputer(rootPath, options);
await computer.computeMetrics();
stopwatch.stop();
@@ -176,25 +170,25 @@
help: 'The number of characters to include in the prefix. Each '
'completion will be requested this many characters in from the '
'start of the token being completed.')
- ..addFlag(CompletionMetricsOptions.PRINT_MISSED_COMPLETION_DETAILS,
+ ..addFlag(CompletionMetricsQualityOptions.PRINT_MISSED_COMPLETION_DETAILS,
defaultsTo: false,
help:
'Print detailed information every time a completion request fails '
'to produce a suggestions matching the expected suggestion.',
negatable: false)
- ..addFlag(CompletionMetricsOptions.PRINT_MISSED_COMPLETION_SUMMARY,
+ ..addFlag(CompletionMetricsQualityOptions.PRINT_MISSED_COMPLETION_SUMMARY,
defaultsTo: false,
help: 'Print summary information about the times that a completion '
'request failed to produce a suggestions matching the expected '
'suggestion.',
negatable: false)
- ..addFlag(CompletionMetricsOptions.PRINT_MISSING_INFORMATION,
+ ..addFlag(CompletionMetricsQualityOptions.PRINT_MISSING_INFORMATION,
defaultsTo: false,
help: 'Print information about places where no completion location was '
'computed and about information that is missing in the completion '
'tables.',
negatable: false)
- ..addFlag(CompletionMetricsOptions.PRINT_MRR_BY_LOCATION,
+ ..addFlag(CompletionMetricsQualityOptions.PRINT_MRR_BY_LOCATION,
defaultsTo: false,
help:
'Print information about the mrr score achieved at each completion '
@@ -202,7 +196,7 @@
'score by pointing out the locations that are causing the biggest '
'impact.',
negatable: false)
- ..addFlag(CompletionMetricsOptions.PRINT_SHADOWED_COMPLETION_DETAILS,
+ ..addFlag(CompletionMetricsQualityOptions.PRINT_SHADOWED_COMPLETION_DETAILS,
defaultsTo: false,
help: 'Print detailed information every time a completion request '
'produces a suggestion whose name matches the expected suggestion '
@@ -213,7 +207,7 @@
help: 'Print information about the completion requests that were the '
'slowest to return suggestions.',
negatable: false)
- ..addFlag(CompletionMetricsOptions.PRINT_WORST_RESULTS,
+ ..addFlag(CompletionMetricsQualityOptions.PRINT_WORST_RESULTS,
defaultsTo: false,
help: 'Print information about the completion requests that had the '
'worst mrr scores.',
@@ -303,7 +297,7 @@
}
/// A wrapper for the collection of [Counter] and [MeanReciprocalRankComputer]
-/// objects for a run of [CompletionMetricsComputer].
+/// objects for a run of [CompletionQualityMetricsComputer].
class CompletionMetrics {
/// The maximum number of slowest results to collect.
static const maxSlowestResults = 100;
@@ -652,25 +646,122 @@
}
}
+/// The options specified on the command-line.
+class CompletionMetricsQualityOptions extends CompletionMetricsOptions {
+ /// A flag that causes detailed information to be printed every time a
+ /// completion request fails to produce a suggestions matching the expected
+ /// suggestion.
+ static const String PRINT_MISSED_COMPLETION_DETAILS =
+ 'print-missed-completion-details';
+
+ /// A flag that causes summary information to be printed about the times that
+ /// a completion request failed to produce a suggestions matching the expected
+ /// suggestion.
+ static const String PRINT_MISSED_COMPLETION_SUMMARY =
+ 'print-missed-completion-summary';
+
+ /// A flag that causes information to be printed about places where no
+ /// completion location was computed and about information that's missing in
+ /// the completion tables.
+ static const String PRINT_MISSING_INFORMATION = 'print-missing-information';
+
+ /// A flag that causes information to be printed about the mrr score achieved
+ /// at each completion location.
+ static const String PRINT_MRR_BY_LOCATION = 'print-mrr-by-location';
+
+ /// A flag that causes detailed information to be printed every time a
+ /// completion request produce a suggestions whose name matches the expected
+ /// suggestion but that is referencing a different element (one that's
+ /// shadowed by the correct element).
+ static const String PRINT_SHADOWED_COMPLETION_DETAILS =
+ 'print-shadowed-completion-details';
+
+ /// A flag that causes information to be printed about the completion requests
+ /// that had the worst mrr scores.
+ static const String PRINT_WORST_RESULTS = 'print-worst-results';
+
+ /// A flag indicating whether information should be printed every time a
+ /// completion request fails to produce a suggestions matching the expected
+ /// suggestion.
+ final bool printMissedCompletionDetails;
+
+ /// A flag indicating whether information should be printed every time a
+ /// completion request fails to produce a suggestions matching the expected
+ /// suggestion.
+ final bool printMissedCompletionSummary;
+
+ /// A flag indicating whether information should be printed about places where
+ /// no completion location was computed and about information that's missing
+ /// in the completion tables.
+ final bool printMissingInformation;
+
+ /// A flag indicating whether information should be printed about the mrr
+ /// score achieved at each completion location.
+ final bool printMrrByLocation;
+
+ /// A flag indicating whether information should be printed every time a
+ /// completion request fails to produce a suggestions matching the expected
+ /// suggestion.
+ final bool printShadowedCompletionDetails;
+
+ /// A flag indicating whether information should be printed about the
+ /// completion requests that had the worst mrr scores.
+ final bool printWorstResults;
+
+ CompletionMetricsQualityOptions(super.results)
+ : printMissedCompletionDetails =
+ results[PRINT_MISSED_COMPLETION_DETAILS] as bool,
+ printMissedCompletionSummary =
+ results[PRINT_MISSED_COMPLETION_SUMMARY] as bool,
+ printMissingInformation = results[PRINT_MISSING_INFORMATION] as bool,
+ printMrrByLocation = results[PRINT_MRR_BY_LOCATION] as bool,
+ printShadowedCompletionDetails =
+ results[PRINT_SHADOWED_COMPLETION_DETAILS] as bool,
+ printWorstResults = results[PRINT_WORST_RESULTS] as bool;
+}
+
/// This is the main metrics computer class for code completions. After the
/// object is constructed, [computeCompletionMetrics] is executed to do analysis
/// and print a summary of the metrics gathered from the completion tests.
-class CompletionMetricsComputer {
- final String rootPath;
-
- final CompletionMetricsOptions options;
-
- late ResolvedUnitResult _resolvedUnitResult;
-
+class CompletionQualityMetricsComputer extends CompletionMetricsComputer {
/// A list of the metrics to be computed.
final List<CompletionMetrics> targetMetrics = [];
- final OverlayResourceProvider _provider =
- OverlayResourceProvider(PhysicalResourceProvider.INSTANCE);
+ DeclarationsTracker? _declarationsTracker;
- int overlayModificationStamp = 0;
+ protocol.CompletionAvailableSuggestionsParams? _availableSuggestionsParams;
- CompletionMetricsComputer(this.rootPath, this.options);
+ CompletionQualityMetricsComputer(
+ super.rootPath, CompletionMetricsQualityOptions super.options);
+
+ @override
+ CompletionMetricsQualityOptions get options =>
+ super.options as CompletionMetricsQualityOptions;
+
+ @override
+ Future<void> applyOverlay(
+ AnalysisContext context,
+ String filePath,
+ ExpectedCompletion expectedCompletion,
+ ) async {
+ // If an overlay option is being used, compute the overlay file, and
+ // have the context reanalyze the file
+ if (options.overlay != CompletionMetricsOptions.OVERLAY_NONE) {
+ var overlayContents = CompletionMetricsComputer.getOverlayContents(
+ resolvedUnitResult.content,
+ expectedCompletion,
+ options.overlay,
+ options.prefixLength);
+
+ provider.setOverlay(filePath,
+ content: overlayContents,
+ modificationStamp: overlayModificationStamp++);
+ context.changeFile(filePath);
+ await context.applyPendingFileChanges();
+ resolvedUnitResult = await context.currentSession
+ .getResolvedUnit(filePath) as ResolvedUnitResult;
+ }
+ }
/// Compare the metrics when each feature is used in isolation.
void compareIndividualFeatures({bool availableSuggestions = false}) {
@@ -713,6 +804,7 @@
}
}
+ @override
Future<void> computeMetrics() async {
// To compare two or more changes to completions, add a `CompletionMetrics`
// object with enable and disable functions to the list of `targetMetrics`.
@@ -734,13 +826,69 @@
// targetMetrics.add(CompletionMetrics('new protocol',
// availableSuggestions: false, useNewProtocol: true));
- final collection = AnalysisContextCollectionImpl(
- includedPaths: [rootPath],
- resourceProvider: PhysicalResourceProvider.INSTANCE,
- );
- for (var context in collection.contexts) {
- await _computeInContext(context.contextRoot);
+ await super.computeMetrics();
+ }
+
+ @override
+ Future<void> computeSuggestionsAndMetrics(
+ ExpectedCompletion expectedCompletion,
+ AnalysisContext context,
+ DocumentationCache documentationCache,
+ ) async {
+ // As this point the completion suggestions are computed,
+ // and results are collected with varying settings for
+ // comparison:
+
+ Future<int> handleExpectedCompletion({
+ required MetricsSuggestionListener listener,
+ required CompletionMetrics metrics,
+ }) async {
+ var stopwatch = Stopwatch()..start();
+ var request = DartCompletionRequest.forResolvedUnit(
+ resolvedUnit: resolvedUnitResult,
+ offset: expectedCompletion.offset,
+ documentationCache: documentationCache,
+ );
+
+ var opType = OpType.forCompletion(request.target, request.offset);
+ var suggestions = await _computeCompletionSuggestions(
+ listener,
+ OperationPerformanceImpl('<root>'),
+ request,
+ metrics.availableSuggestions ? _declarationsTracker : null,
+ metrics.availableSuggestions ? _availableSuggestionsParams : null,
+ metrics.useNewProtocol ? NotImportedSuggestions() : null,
+ );
+ stopwatch.stop();
+
+ return gatherMetricsForSuggestions(
+ request,
+ listener,
+ expectedCompletion,
+ opType.completionLocation,
+ suggestions,
+ metrics,
+ stopwatch.elapsedMilliseconds);
}
+
+ var bestRank = -1;
+ var bestName = '';
+ var defaultTag = getCurrentTag();
+ for (var metrics in targetMetrics) {
+ // Compute the completions.
+ metrics.enable();
+ metrics.userTag.makeCurrent();
+ var listener = MetricsSuggestionListener();
+ var rank =
+ await handleExpectedCompletion(listener: listener, metrics: metrics);
+ if (bestRank < 0 || rank < bestRank) {
+ bestRank = rank;
+ bestName = metrics.name;
+ }
+ defaultTag.makeCurrent();
+ metrics.disable();
+ }
+ rankComparison.count(bestName);
}
/// Gathers various metrics for the completion [request] which resulted in
@@ -1247,6 +1395,32 @@
}
}
+ @override
+ void removeOverlay(String filePath) {
+ // If an overlay option is being used, remove the overlay applied
+ // earlier.
+ if (options.overlay != CompletionMetricsOptions.OVERLAY_NONE) {
+ provider.removeOverlay(filePath);
+ }
+ }
+
+ @override
+ void setupForResolution(AnalysisContext context) {
+ if (targetMetrics.any((metrics) => metrics.availableSuggestions)) {
+ var declarationsTracker = DeclarationsTracker(
+ MemoryByteStore(), PhysicalResourceProvider.INSTANCE);
+ declarationsTracker.addContext(context);
+ while (declarationsTracker.hasWork) {
+ declarationsTracker.doWork();
+ }
+
+ // Have the [AvailableDeclarationsSet] computed to use later.
+ _availableSuggestionsParams = createCompletionAvailableSuggestions(
+ declarationsTracker.allLibraries.toList(), []);
+ _declarationsTracker = declarationsTracker;
+ }
+ }
+
int _computeCharsBeforeTop(ExpectedCompletion target,
List<protocol.CompletionSuggestion> suggestions,
{int minRank = 1}) {
@@ -1358,172 +1532,6 @@
return suggestions;
}
- /// Compute the metrics for the files in the context [root], creating a
- /// separate context collection to prevent accumulating memory. The metrics
- /// should be captured in the [collector].
- Future<void> _computeInContext(ContextRoot root) async {
- // Create a new collection to avoid consuming large quantities of memory.
- final collection = AnalysisContextCollectionImpl(
- includedPaths: root.includedPaths.toList(),
- excludedPaths: root.excludedPaths.toList(),
- resourceProvider: _provider,
- );
-
- var context = collection.contexts[0];
-
- DeclarationsTracker? declarationsTracker;
- protocol.CompletionAvailableSuggestionsParams? availableSuggestionsParams;
- if (targetMetrics.any((metrics) => metrics.availableSuggestions)) {
- declarationsTracker = DeclarationsTracker(
- MemoryByteStore(), PhysicalResourceProvider.INSTANCE);
- declarationsTracker.addContext(context);
- while (declarationsTracker.hasWork) {
- declarationsTracker.doWork();
- }
-
- // Have the [AvailableDeclarationsSet] computed to use later.
- availableSuggestionsParams = createCompletionAvailableSuggestions(
- declarationsTracker.allLibraries.toList(), []);
- }
-
- // Loop through each file, resolve the file and call
- // [forEachExpectedCompletion].
-
- var ansi = Ansi(Ansi.terminalSupportsAnsi);
- var logger = Logger.standard(ansi: ansi);
- var analyzedFileCount = context.contextRoot.analyzedFiles().length;
- logger.write('Computing completions at root: ${root.root.path} '
- '($analyzedFileCount files)\n');
-
- logger.write('Resolving...\n');
- var progress = _ProgressBar(logger, analyzedFileCount);
-
- var dartdocDirectiveInfo = DartdocDirectiveInfo();
- var documentationCache = DocumentationCache(dartdocDirectiveInfo);
- var results = <ResolvedUnitResult>[];
- var pathContext = context.contextRoot.resourceProvider.pathContext;
- for (var filePath in context.contextRoot.analyzedFiles()) {
- if (file_paths.isDart(pathContext, filePath)) {
- try {
- var result = await context.currentSession.getResolvedUnit(filePath)
- as ResolvedUnitResult;
-
- var analysisError = getFirstErrorOrNull(result);
- if (analysisError != null) {
- progress.clear();
- print('File $filePath skipped due to errors such as:');
- print(' ${analysisError.toString()}');
- print('');
- continue;
- } else {
- results.add(result);
- documentationCache.cacheFromResult(result);
- }
- } catch (exception, stackTrace) {
- progress.clear();
- print('Exception caught analyzing: $filePath');
- print(exception.toString());
- print(stackTrace);
- }
- }
- progress.tick();
- }
- progress.complete();
-
- logger.write('Analyzing completion suggestions...\n');
- progress = _ProgressBar(logger, results.length);
- for (var result in results) {
- _resolvedUnitResult = result;
- var filePath = result.path;
- // Use the ExpectedCompletionsVisitor to compute the set of expected
- // completions for this CompilationUnit.
- final visitor =
- ExpectedCompletionsVisitor(result, caretOffset: options.prefixLength);
- _resolvedUnitResult.unit.accept(visitor);
-
- for (var expectedCompletion in visitor.expectedCompletions) {
- var resolvedUnitResult = _resolvedUnitResult;
-
- // If an overlay option is being used, compute the overlay file, and
- // have the context reanalyze the file
- if (options.overlay != CompletionMetricsOptions.OVERLAY_NONE) {
- var overlayContents = _getOverlayContents(
- _resolvedUnitResult.content, expectedCompletion);
-
- _provider.setOverlay(filePath,
- content: overlayContents,
- modificationStamp: overlayModificationStamp++);
- context.changeFile(filePath);
- await context.applyPendingFileChanges();
- resolvedUnitResult = await context.currentSession
- .getResolvedUnit(filePath) as ResolvedUnitResult;
- }
-
- // As this point the completion suggestions are computed,
- // and results are collected with varying settings for
- // comparison:
-
- Future<int> handleExpectedCompletion(
- {required MetricsSuggestionListener listener,
- required CompletionMetrics metrics}) async {
- var stopwatch = Stopwatch()..start();
- var request = DartCompletionRequest.forResolvedUnit(
- resolvedUnit: resolvedUnitResult,
- offset: expectedCompletion.offset,
- documentationCache: documentationCache,
- );
-
- var opType = OpType.forCompletion(request.target, request.offset);
- var suggestions = await _computeCompletionSuggestions(
- listener,
- OperationPerformanceImpl('<root>'),
- request,
- metrics.availableSuggestions ? declarationsTracker : null,
- metrics.availableSuggestions ? availableSuggestionsParams : null,
- metrics.useNewProtocol ? NotImportedSuggestions() : null,
- );
- stopwatch.stop();
-
- return gatherMetricsForSuggestions(
- request,
- listener,
- expectedCompletion,
- opType.completionLocation,
- suggestions,
- metrics,
- stopwatch.elapsedMilliseconds);
- }
-
- var bestRank = -1;
- var bestName = '';
- var defaultTag = getCurrentTag();
- for (var metrics in targetMetrics) {
- // Compute the completions.
- metrics.enable();
- metrics.userTag.makeCurrent();
- var listener = MetricsSuggestionListener();
- var rank = await handleExpectedCompletion(
- listener: listener, metrics: metrics);
- if (bestRank < 0 || rank < bestRank) {
- bestRank = rank;
- bestName = metrics.name;
- }
- defaultTag.makeCurrent();
- metrics.disable();
- }
- rankComparison.count(bestName);
-
- // If an overlay option is being used, remove the overlay applied
- // earlier.
- if (options.overlay != CompletionMetricsOptions.OVERLAY_NONE) {
- _provider.removeOverlay(filePath);
- }
- }
- progress.tick();
- }
- progress.complete();
- }
-
List<protocol.CompletionSuggestion> _filterSuggestions(
String prefix, List<protocol.CompletionSuggestion> suggestions) {
// TODO(brianwilkerson) Replace this with a more realistic filtering
@@ -1533,32 +1541,6 @@
.toList();
}
- String _getOverlayContents(
- String contents, ExpectedCompletion expectedCompletion) {
- assert(contents.isNotEmpty);
- var offset = expectedCompletion.offset;
- var length = expectedCompletion.syntacticEntity.length;
- assert(offset >= 0);
- assert(length > 0);
- var tokenEndOffset = offset + length;
- if (length >= options.prefixLength) {
- // Rather than removing the whole token, remove the characters after
- // the given prefix length.
- offset += options.prefixLength;
- }
- if (options.overlay == CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN) {
- return contents.substring(0, offset) + contents.substring(tokenEndOffset);
- } else if (options.overlay ==
- CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE) {
- return contents.substring(0, offset);
- } else {
- var removeToken = CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN;
- var removeRest = CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE;
- throw Exception('\'_getOverlayContents\' called with option other than'
- '$removeToken and $removeRest: ${options.overlay}');
- }
- }
-
void _printWorstResults(String title, List<CompletionResult> worstResults) {
List<String> suggestionRow(int rank, SuggestionData data) {
var suggestion = data.suggestion;
@@ -1641,18 +1623,6 @@
}
}
- /// Given some [ResolvedUnitResult] return the first error of high severity
- /// if such an error exists, `null` otherwise.
- static err.AnalysisError? getFirstErrorOrNull(
- ResolvedUnitResult resolvedUnitResult) {
- for (var error in resolvedUnitResult.errors) {
- if (error.severity == Severity.error) {
- return error;
- }
- }
- return null;
- }
-
/// Returns a [Place] indicating the position of [expectedCompletion] in
/// [suggestions].
///
@@ -1669,132 +1639,6 @@
}
}
-/// The options specified on the command-line.
-class CompletionMetricsOptions {
- /// An option to control whether and how overlays should be produced.
- static const String OVERLAY = 'overlay';
-
- /// A mode indicating that no overlays should be produced.
- static const String OVERLAY_NONE = 'none';
-
- /// A mode indicating that everything from the completion offset to the end of
- /// the file should be removed.
- static const String OVERLAY_REMOVE_REST_OF_FILE = 'remove-rest-of-file';
-
- /// A mode indicating that the token whose offset is the same as the
- /// completion offset should be removed.
- static const String OVERLAY_REMOVE_TOKEN = 'remove-token';
-
- /// An option controlling how long of a prefix should be used.
- ///
- /// This affects the offset of the completion request, and how much content is
- /// removed in each of the overlay modes.
- static const String PREFIX_LENGTH = 'prefix-length';
-
- /// A flag that causes detailed information to be printed every time a
- /// completion request fails to produce a suggestions matching the expected
- /// suggestion.
- static const String PRINT_MISSED_COMPLETION_DETAILS =
- 'print-missed-completion-details';
-
- /// A flag that causes summary information to be printed about the times that
- /// a completion request failed to produce a suggestions matching the expected
- /// suggestion.
- static const String PRINT_MISSED_COMPLETION_SUMMARY =
- 'print-missed-completion-summary';
-
- /// A flag that causes information to be printed about places where no
- /// completion location was computed and about information that's missing in
- /// the completion tables.
- static const String PRINT_MISSING_INFORMATION = 'print-missing-information';
-
- /// A flag that causes information to be printed about the mrr score achieved
- /// at each completion location.
- static const String PRINT_MRR_BY_LOCATION = 'print-mrr-by-location';
-
- /// A flag that causes detailed information to be printed every time a
- /// completion request produce a suggestions whose name matches the expected
- /// suggestion but that is referencing a different element (one that's
- /// shadowed by the correct element).
- static const String PRINT_SHADOWED_COMPLETION_DETAILS =
- 'print-shadowed-completion-details';
-
- /// A flag that causes information to be printed about the completion requests
- /// that were the slowest to return suggestions.
- static const String PRINT_SLOWEST_RESULTS = 'print-slowest-results';
-
- /// A flag that causes information to be printed about the completion requests
- /// that had the worst mrr scores.
- static const String PRINT_WORST_RESULTS = 'print-worst-results';
-
- /// The overlay mode that should be used.
- final String overlay;
-
- final int prefixLength;
-
- /// A flag indicating whether information should be printed every time a
- /// completion request fails to produce a suggestions matching the expected
- /// suggestion.
- final bool printMissedCompletionDetails;
-
- /// A flag indicating whether information should be printed every time a
- /// completion request fails to produce a suggestions matching the expected
- /// suggestion.
- final bool printMissedCompletionSummary;
-
- /// A flag indicating whether information should be printed about places where
- /// no completion location was computed and about information that's missing
- /// in the completion tables.
- final bool printMissingInformation;
-
- /// A flag indicating whether information should be printed about the mrr
- /// score achieved at each completion location.
- final bool printMrrByLocation;
-
- /// A flag indicating whether information should be printed every time a
- /// completion request fails to produce a suggestions matching the expected
- /// suggestion.
- final bool printShadowedCompletionDetails;
-
- /// A flag indicating whether information should be printed about the
- /// completion requests that were the slowest to return suggestions.
- final bool printSlowestResults;
-
- /// A flag indicating whether information should be printed about the
- /// completion requests that had the worst mrr scores.
- final bool printWorstResults;
-
- factory CompletionMetricsOptions(results) {
- return CompletionMetricsOptions._(
- overlay: results[OVERLAY] as String,
- prefixLength: int.parse(results[PREFIX_LENGTH] as String),
- printMissedCompletionDetails:
- results[PRINT_MISSED_COMPLETION_DETAILS] as bool,
- printMissedCompletionSummary:
- results[PRINT_MISSED_COMPLETION_SUMMARY] as bool,
- printMissingInformation: results[PRINT_MISSING_INFORMATION] as bool,
- printMrrByLocation: results[PRINT_MRR_BY_LOCATION] as bool,
- printShadowedCompletionDetails:
- results[PRINT_SHADOWED_COMPLETION_DETAILS] as bool,
- printSlowestResults: results[PRINT_SLOWEST_RESULTS] as bool,
- printWorstResults: results[PRINT_WORST_RESULTS] as bool);
- }
-
- CompletionMetricsOptions._(
- {required this.overlay,
- required this.prefixLength,
- required this.printMissedCompletionDetails,
- required this.printMissedCompletionSummary,
- required this.printMissingInformation,
- required this.printMrrByLocation,
- required this.printShadowedCompletionDetails,
- required this.printSlowestResults,
- required this.printWorstResults})
- : assert(overlay == OVERLAY_NONE ||
- overlay == OVERLAY_REMOVE_TOKEN ||
- overlay == OVERLAY_REMOVE_REST_OF_FILE);
-}
-
/// The result of a single completion.
class CompletionResult {
final Place place;
@@ -2121,84 +1965,6 @@
}
}
-/// A facility for drawing a progress bar in the terminal.
-///
-/// The bar is instantiated with the total number of "ticks" to be completed,
-/// and progress is made by calling [tick]. The bar is drawn across one entire
-/// line, like so:
-///
-/// [---------- ]
-///
-/// The hyphens represent completed progress, and the whitespace represents
-/// remaining progress.
-///
-/// If there is no terminal, the progress bar will not be drawn.
-class _ProgressBar {
- /// Whether the progress bar should be drawn.
- late bool _shouldDrawProgress;
-
- /// The width of the terminal, in terms of characters.
- late int _width;
-
- final Logger _logger;
-
- /// The inner width of the terminal, in terms of characters.
- ///
- /// This represents the number of characters available for drawing progress.
- late int _innerWidth;
-
- final int _totalTickCount;
-
- int _tickCount = 0;
-
- _ProgressBar(this._logger, this._totalTickCount) {
- if (!stdout.hasTerminal) {
- _shouldDrawProgress = false;
- } else {
- _shouldDrawProgress = true;
- _width = stdout.terminalColumns;
- // Inclusion of the percent indicator assumes a terminal width of at least
- // 12 (2 brackets + 1 space + 2 parenthesis characters + 3 digits +
- // 1 period + 2 digits + 1 '%' character).
- _innerWidth = stdout.terminalColumns - 12;
- _logger.write('[${' ' * _innerWidth}]');
- }
- }
-
- /// Clears the progress bar from the terminal, allowing other logging to be
- /// printed.
- void clear() {
- if (!_shouldDrawProgress) {
- return;
- }
- _logger.write('\r${' ' * _width}\r');
- }
-
- /// Draws the progress bar as complete, and print two newlines.
- void complete() {
- if (!_shouldDrawProgress) {
- return;
- }
- _logger.write('\r[${'-' * _innerWidth}]\n\n');
- }
-
- /// Progresses the bar by one tick.
- void tick() {
- if (!_shouldDrawProgress) {
- return;
- }
- _tickCount++;
- var fractionComplete =
- math.max(0, _tickCount * _innerWidth ~/ _totalTickCount - 1);
- // The inner space consists of hyphens, one spinner character, and spaces.
- var remaining = _innerWidth - fractionComplete - 1;
- var spinner = AnsiProgress.kAnimationItems[_tickCount % 4];
- var pctComplete = (_tickCount * 100 / _totalTickCount).toStringAsFixed(2);
- _logger.write(
- '\r[${'-' * fractionComplete}$spinner${' ' * remaining}] ($pctComplete%)');
- }
-}
-
extension on CompletionGroup {
String get name {
switch (this) {
diff --git a/pkg/analysis_server/tool/code_completion/completion_metrics_base.dart b/pkg/analysis_server/tool/code_completion/completion_metrics_base.dart
new file mode 100644
index 0000000..24cf9c6
--- /dev/null
+++ b/pkg/analysis_server/tool/code_completion/completion_metrics_base.dart
@@ -0,0 +1,333 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io' show stdout;
+import 'dart:math' as math;
+
+import 'package:analysis_server/src/services/completion/dart/documentation_cache.dart';
+import 'package:analyzer/dart/analysis/analysis_context.dart';
+import 'package:analyzer/dart/analysis/context_root.dart';
+import 'package:analyzer/dart/analysis/results.dart';
+import 'package:analyzer/diagnostic/diagnostic.dart';
+import 'package:analyzer/error/error.dart' as err;
+import 'package:analyzer/file_system/overlay_file_system.dart';
+import 'package:analyzer/file_system/physical_file_system.dart';
+import 'package:analyzer/src/dart/analysis/analysis_context_collection.dart';
+import 'package:analyzer/src/dartdoc/dartdoc_directive_info.dart';
+import 'package:analyzer/src/util/file_paths.dart' as file_paths;
+import 'package:args/args.dart';
+import 'package:cli_util/cli_logging.dart';
+
+import 'visitors.dart';
+
+final logger = Logger.standard(ansi: Ansi(Ansi.terminalSupportsAnsi));
+
+/// This is the main metrics computer class for code completions. After the
+/// object is constructed, [computeCompletionMetrics] is executed to do analysis
+/// and print a summary of the metrics gathered from the completion tests.
+abstract class CompletionMetricsComputer {
+ final String rootPath;
+
+ final CompletionMetricsOptions options;
+
+ late ResolvedUnitResult resolvedUnitResult;
+
+ final OverlayResourceProvider provider =
+ OverlayResourceProvider(PhysicalResourceProvider.INSTANCE);
+
+ int overlayModificationStamp = 0;
+
+ CompletionMetricsComputer(this.rootPath, this.options);
+
+ /// Applies an overlay in [filePath] at [expectedCompletion].
+ Future<void> applyOverlay(AnalysisContext context, String filePath,
+ ExpectedCompletion expectedCompletion);
+
+ /// Compute the metrics for the files in the context [root], creating a
+ /// separate context collection to prevent accumulating memory. The metrics
+ /// should be captured in the [collector].
+ Future<void> computeInContext(ContextRoot root) async {
+ // Create a new collection to avoid consuming large quantities of memory.
+ final collection = AnalysisContextCollectionImpl(
+ includedPaths: root.includedPaths.toList(),
+ excludedPaths: root.excludedPaths.toList(),
+ resourceProvider: provider,
+ );
+
+ var context = collection.contexts[0];
+
+ setupForResolution(context);
+
+ logger.write('Computing completions at root: ${root.root.path}\n');
+ var documentationCache = DocumentationCache(DartdocDirectiveInfo());
+
+ var results = await resolveAnalyzedFiles(
+ context: context,
+ documentationCache: documentationCache,
+ );
+
+ logger.write('Analyzing completion suggestions...\n');
+ var progress = ProgressBar(logger, results.length);
+ for (var result in results) {
+ resolvedUnitResult = result;
+ var filePath = result.path;
+ // Use the ExpectedCompletionsVisitor to compute the set of expected
+ // completions for this CompilationUnit.
+ final visitor =
+ ExpectedCompletionsVisitor(result, caretOffset: options.prefixLength);
+ resolvedUnitResult.unit.accept(visitor);
+
+ for (var expectedCompletion in visitor.expectedCompletions) {
+ await applyOverlay(context, filePath, expectedCompletion);
+
+ await computeSuggestionsAndMetrics(
+ expectedCompletion,
+ context,
+ documentationCache,
+ );
+
+ removeOverlay(filePath);
+ }
+ progress.tick();
+ }
+ progress.complete();
+ }
+
+ Future<void> computeMetrics() async {
+ final collection = AnalysisContextCollectionImpl(
+ includedPaths: [rootPath],
+ resourceProvider: PhysicalResourceProvider.INSTANCE,
+ );
+ for (final context in collection.contexts) {
+ await computeInContext(context.contextRoot);
+ }
+ }
+
+ /// Computes suggestions for [expectedCompletion] and computes metrics from
+ /// the resulting suggestions.
+ Future<void> computeSuggestionsAndMetrics(
+ ExpectedCompletion expectedCompletion,
+ AnalysisContext context,
+ DocumentationCache documentationCache,
+ );
+
+ /// Removes the overlay which has been applied to [filePath].
+ void removeOverlay(String filePath);
+
+ /// Resolves all analyzed files within [context].
+ Future<List<ResolvedUnitResult>> resolveAnalyzedFiles({
+ required AnalysisContext context,
+ required DocumentationCache documentationCache,
+ }) async {
+ final analyzedFileCount = context.contextRoot.analyzedFiles().length;
+ logger.write('Resolving $analyzedFileCount files...\n');
+
+ final progress = ProgressBar(logger, analyzedFileCount);
+ final results = <ResolvedUnitResult>[];
+ final pathContext = context.contextRoot.resourceProvider.pathContext;
+ for (final filePath in context.contextRoot.analyzedFiles()) {
+ if (file_paths.isDart(pathContext, filePath)) {
+ try {
+ final result = await context.currentSession.getResolvedUnit(filePath)
+ as ResolvedUnitResult;
+
+ final analysisError = getFirstErrorOrNull(result);
+ if (analysisError != null) {
+ progress.clear();
+ print('File $filePath skipped due to errors such as:');
+ print(' ${analysisError.toString()}');
+ print('');
+ continue;
+ } else {
+ results.add(result);
+ documentationCache.cacheFromResult(result);
+ }
+ } catch (exception, stackTrace) {
+ progress.clear();
+ print('Exception caught analyzing: $filePath');
+ print(exception.toString());
+ print(stackTrace);
+ }
+ }
+ progress.tick();
+ }
+ progress.complete();
+ return results;
+ }
+
+ /// Performs setup tasks with [context] before resolution.
+ void setupForResolution(AnalysisContext context);
+
+ /// Given some [ResolvedUnitResult] returns the first error of high severity
+ /// if such an error exists, `null` otherwise.
+ static err.AnalysisError? getFirstErrorOrNull(
+ ResolvedUnitResult resolvedUnitResult) {
+ for (final error in resolvedUnitResult.errors) {
+ if (error.severity == Severity.error) {
+ return error;
+ }
+ }
+ return null;
+ }
+
+ /// Gets overlay contents for [contents], applying a change at
+ /// [expectedCompletion] with [prefixLength], according to [overlay], one of
+ /// the [CompletionMetricsOptions].
+ static String getOverlayContents(
+ String contents,
+ ExpectedCompletion expectedCompletion,
+ // TODO(srawlins): Replace this with an enum.
+ String overlay,
+ int prefixLength,
+ ) {
+ assert(contents.isNotEmpty);
+ var offset = expectedCompletion.offset;
+ final length = expectedCompletion.syntacticEntity.length;
+ assert(offset >= 0);
+ assert(length > 0);
+ var tokenEndOffset = offset + length;
+ if (length >= prefixLength) {
+ // Rather than removing the whole token, remove the characters after
+ // the given prefix length.
+ offset += prefixLength;
+ }
+ if (overlay == CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN) {
+ return contents.substring(0, offset) + contents.substring(tokenEndOffset);
+ } else if (overlay ==
+ CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE) {
+ return contents.substring(0, offset);
+ } else {
+ final removeToken = CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN;
+ final removeRest = CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE;
+ throw Exception('\'getOverlayContents\' called with option other than'
+ '$removeToken and $removeRest: $overlay');
+ }
+ }
+}
+
+/// The options specified on the command-line.
+class CompletionMetricsOptions {
+ /// An option to control whether and how overlays should be produced.
+ static const String OVERLAY = 'overlay';
+
+ /// A mode indicating that no overlays should be produced.
+ /// TODO(srawlins): Replace this and the other two overlay values with enums.
+ static const String OVERLAY_NONE = 'none';
+
+ /// A mode indicating that everything from the completion offset to the end of
+ /// the file should be removed.
+ static const String OVERLAY_REMOVE_REST_OF_FILE = 'remove-rest-of-file';
+
+ /// A mode indicating that the token whose offset is the same as the
+ /// completion offset should be removed.
+ static const String OVERLAY_REMOVE_TOKEN = 'remove-token';
+
+ /// An option controlling how long of a prefix should be used.
+ ///
+ /// This affects the offset of the completion request, and how much content is
+ /// removed in each of the overlay modes.
+ static const String PREFIX_LENGTH = 'prefix-length';
+
+ /// A flag that causes information to be printed about the completion requests
+ /// that were the slowest to return suggestions.
+ static const String PRINT_SLOWEST_RESULTS = 'print-slowest-results';
+
+ /// The overlay mode that should be used.
+ /// TODO(srawlins): Replace this with an enum.
+ final String overlay;
+
+ final int prefixLength;
+
+ /// A flag indicating whether information should be printed about the
+ /// completion requests that were the slowest to return suggestions.
+ final bool printSlowestResults;
+
+ CompletionMetricsOptions(ArgResults results)
+ : overlay = results[OVERLAY] as String,
+ prefixLength = int.parse(results[PREFIX_LENGTH] as String),
+ printSlowestResults = results[PRINT_SLOWEST_RESULTS] as bool {
+ assert(overlay == OVERLAY_NONE ||
+ overlay == OVERLAY_REMOVE_TOKEN ||
+ overlay == OVERLAY_REMOVE_REST_OF_FILE);
+ }
+}
+
+/// A facility for drawing a progress bar in the terminal.
+///
+/// The bar is instantiated with the total number of "ticks" to be completed,
+/// and progress is made by calling [tick]. The bar is drawn across one entire
+/// line, like so:
+///
+/// [---------- ]
+///
+/// The hyphens represent completed progress, and the whitespace represents
+/// remaining progress.
+///
+/// If there is no terminal, the progress bar will not be drawn.
+class ProgressBar {
+ /// Whether the progress bar should be drawn.
+ late bool _shouldDrawProgress;
+
+ /// The width of the terminal, in terms of characters.
+ late int _width;
+
+ final Logger _logger;
+
+ /// The inner width of the terminal, in terms of characters.
+ ///
+ /// This represents the number of characters available for drawing progress.
+ late int _innerWidth;
+
+ final int _totalTickCount;
+
+ int _tickCount = 0;
+
+ ProgressBar(this._logger, this._totalTickCount) {
+ if (!stdout.hasTerminal) {
+ _shouldDrawProgress = false;
+ } else {
+ _shouldDrawProgress = true;
+ _width = stdout.terminalColumns;
+ // Inclusion of the percent indicator assumes a terminal width of at least
+ // 12 (2 brackets + 1 space + 2 parenthesis characters + 3 digits +
+ // 1 period + 2 digits + 1 '%' character).
+ _innerWidth = stdout.terminalColumns - 12;
+ _logger.write('[${' ' * _innerWidth}]');
+ }
+ }
+
+ /// Clears the progress bar from the terminal, allowing other logging to be
+ /// printed.
+ void clear() {
+ if (!_shouldDrawProgress) {
+ return;
+ }
+ _logger.write('\r${' ' * _width}\r');
+ }
+
+ /// Draws the progress bar as complete, and print two newlines.
+ void complete() {
+ if (!_shouldDrawProgress) {
+ return;
+ }
+ _logger.write('\r[${'-' * _innerWidth}]\n\n');
+ }
+
+ /// Progresses the bar by one tick.
+ void tick() {
+ if (!_shouldDrawProgress) {
+ return;
+ }
+ _tickCount++;
+ final fractionComplete =
+ math.max(0, _tickCount * _innerWidth ~/ _totalTickCount - 1);
+ // The inner space consists of hyphens, one spinner character, spaces, and a
+ // percentage (8 characters).
+ final hyphens = '-' * fractionComplete;
+ final trailingSpace = ' ' * (_innerWidth - fractionComplete - 1);
+ final spinner = AnsiProgress.kAnimationItems[_tickCount % 4];
+ final pctComplete = (_tickCount * 100 / _totalTickCount).toStringAsFixed(2);
+ _logger.write('\r[$hyphens$spinner$trailingSpace] ($pctComplete%)');
+ }
+}
diff --git a/pkg/analysis_server/tool/code_completion/completion_metrics_client.dart b/pkg/analysis_server/tool/code_completion/completion_metrics_client.dart
new file mode 100644
index 0000000..2248b30
--- /dev/null
+++ b/pkg/analysis_server/tool/code_completion/completion_metrics_client.dart
@@ -0,0 +1,592 @@
+// Copyright (c) 2022, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:analysis_server/protocol/protocol_generated.dart';
+import 'package:analysis_server/src/protocol/protocol_internal.dart';
+import 'package:analysis_server/src/server/driver.dart';
+import 'package:analysis_server/src/services/completion/dart/documentation_cache.dart';
+import 'package:analyzer/dart/analysis/analysis_context.dart';
+import 'package:args/args.dart';
+import 'package:path/path.dart' as path;
+
+import 'completion_metrics_base.dart';
+import 'metrics_util.dart';
+import 'output_utilities.dart';
+import 'relevance_table_generator.dart';
+import 'visitors.dart';
+
+Future<void> main(List<String> args) async {
+ var parser = _createArgParser();
+ var result = parser.parse(args);
+
+ if (!_validArguments(parser, result)) {
+ return;
+ }
+
+ var rootPath = result.rest[0];
+ final targets = <Directory>[];
+ if (Directory(rootPath).existsSync()) {
+ targets.add(Directory(rootPath));
+ } else {
+ throw "Directory doesn't exist: $rootPath";
+ }
+
+ var options = CompletionMetricsOptions(result);
+ var stopwatch = Stopwatch()..start();
+ var client = _AnalysisServerClient(Directory(_sdk.sdkPath), targets);
+ CompletionClientMetricsComputer(rootPath, options, client).computeMetrics();
+ stopwatch.stop();
+
+ var duration = Duration(milliseconds: stopwatch.elapsedMilliseconds);
+ print('');
+ print('Metrics computed in $duration');
+}
+
+final _Sdk _sdk = _Sdk._instance;
+
+/// Given a data structure which is a Map of String to dynamic values, returns
+/// the same structure (`Map<String, dynamic>`) with the correct runtime types.
+Map<String, dynamic> _castStringKeyedMap(dynamic untyped) {
+ final Map<dynamic, dynamic> map = untyped! as Map<dynamic, dynamic>;
+ return map.cast<String, dynamic>();
+}
+
+/// Creates a parser that can be used to parse the command-line arguments.
+ArgParser _createArgParser() {
+ return ArgParser(
+ usageLineLength: stdout.hasTerminal ? stdout.terminalColumns : 80)
+ ..addFlag(
+ 'help',
+ abbr: 'h',
+ help: 'Print this help message.',
+ )
+ ..addOption(
+ CompletionMetricsOptions.OVERLAY,
+ allowed: [
+ CompletionMetricsOptions.OVERLAY_NONE,
+ CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN,
+ CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE,
+ ],
+ defaultsTo: CompletionMetricsOptions.OVERLAY_NONE,
+ help: 'Before attempting a completion at the location of each token, the '
+ 'token can be removed, or the rest of the file can be removed to '
+ 'test code completion with diverse methods. The default mode is to '
+ 'complete at the start of the token without modifying the file.',
+ )
+ ..addOption(
+ CompletionMetricsOptions.PREFIX_LENGTH,
+ defaultsTo: '0',
+ help: 'The number of characters to include in the prefix. Each '
+ 'completion will be requested this many characters in from the '
+ 'start of the token being completed.',
+ )
+ ..addFlag(
+ CompletionMetricsOptions.PRINT_SLOWEST_RESULTS,
+ defaultsTo: false,
+ help: 'Print information about the completion requests that were the '
+ 'slowest to return suggestions.',
+ negatable: false,
+ );
+}
+
+/// Prints usage information for this tool.
+void _printUsage(ArgParser parser, {String? error}) {
+ if (error != null) {
+ print(error);
+ print('');
+ }
+ print('usage: dart completion_metrics_client.dart [options] packagePath');
+ print('');
+ print('Compute code completion health metrics.');
+ print('');
+ print(parser.usage);
+}
+
+/// Trims [suffix] from the end of [text].
+String _trimEnd(String text, String suffix) {
+ if (text.endsWith(suffix)) {
+ return text.substring(0, text.length - suffix.length);
+ }
+ return text;
+}
+
+/// Returns `true` if the command-line arguments (represented by the [result]
+/// and parsed by the [parser]) are valid.
+bool _validArguments(ArgParser parser, ArgResults result) {
+ if (result.wasParsed('help')) {
+ _printUsage(parser);
+ return false;
+ } else if (result.rest.length != 1) {
+ _printUsage(parser, error: 'No package path specified.');
+ return false;
+ }
+ return validateDir(parser, result.rest[0]);
+}
+
+class CompletionClientMetricsComputer extends CompletionMetricsComputer {
+ final _AnalysisServerClient client;
+
+ final CompletionMetrics targetMetric = CompletionMetrics();
+
+ final metrics = CompletionMetrics();
+
+ CompletionClientMetricsComputer(super.rootPath, super.options, this.client);
+
+ @override
+ Future<void> applyOverlay(
+ AnalysisContext context,
+ String filePath,
+ ExpectedCompletion expectedCompletion,
+ ) async {
+ // TODO(srawlins): Support overlays.
+ }
+
+ @override
+ Future<void> computeMetrics() async {
+ await client.start();
+ await super.computeMetrics();
+ await client.shutdown();
+
+ // A row containing the name, median, p90, and p95 scores in [computer].
+ List<String> m9095Row(PercentileComputer computer) => [
+ computer.name,
+ computer.median.toString(),
+ computer.p90.toString(),
+ computer.p95.toString(),
+ ];
+
+ var table = [
+ ['', 'median', 'p90', 'p95'],
+ m9095Row(metrics.totalPercentileComputer),
+ m9095Row(metrics.requestResponsePercentileComputer),
+ m9095Row(metrics.decodePercentileComputer),
+ m9095Row(metrics.deserializePercentileComputer),
+ ];
+
+ rightJustifyColumns(table, range(1, table[0].length));
+ printTable(table);
+ }
+
+ @override
+ Future<void> computeSuggestionsAndMetrics(
+ ExpectedCompletion expectedCompletion,
+ AnalysisContext context,
+ DocumentationCache documentationCache,
+ ) async {
+ var stopwatch = Stopwatch()..start();
+ var suggestionsData = await client.requestCompletion(
+ expectedCompletion.filePath, expectedCompletion.offset, 1000);
+ stopwatch.stop();
+ var metadata = suggestionsData.metadata;
+
+ metrics.totalPercentileComputer.addValue(stopwatch.elapsedMilliseconds);
+ metrics.requestResponsePercentileComputer
+ .addValue(metadata.requestResponseDuration);
+ metrics.decodePercentileComputer.addValue(metadata.decodeDuration);
+ metrics.deserializePercentileComputer
+ .addValue(metadata.deserializeDuration);
+ }
+
+ @override
+ void removeOverlay(String filePath) {
+ // TODO(srawlins): Support overlays.
+ }
+
+ @override
+ void setupForResolution(AnalysisContext context) {}
+}
+
+class CompletionMetrics {
+ /// A percentile computer which tracks the total time to create and send a
+ /// completion request, and receive and decode a completion response, using
+ /// 2.000 seconds as the max value to use in percentile calculations.
+ final PercentileComputer totalPercentileComputer =
+ PercentileComputer('ms for total duration', valueLimit: 2000);
+
+ /// A percentile computer which tracks the time to send a completion request,
+ /// and receive a completion response, not including any time to encode or
+ /// decode, using 2.000 seconds as the max value to use in percentile
+ /// calculations.
+ final PercentileComputer requestResponsePercentileComputer =
+ PercentileComputer('ms for request/response duration', valueLimit: 2000);
+
+ /// A percentile computer which tracks the time to decode each completion
+ /// response into JSON, using 2.000 seconds as the max value to use in
+ /// percentile calculations.
+ final PercentileComputer decodePercentileComputer =
+ PercentileComputer('ms for decode duration', valueLimit: 2000);
+
+ /// A percentile computer which tracks the time to deserialize each completion
+ /// response JSON into Dart objects, using 2.000 seconds as the max value to
+ /// use in percentile calculations.
+ final PercentileComputer deserializePercentileComputer =
+ PercentileComputer('ms for deserialize duration', valueLimit: 2000);
+}
+
+/// A client for communicating with the analysis server over stdin/stdout.
+class _AnalysisServerClient {
+ // This class is copied from package:dartdev/src/analysis_server.dart and
+ // stripped.
+
+ final Directory sdkPath;
+ final List<FileSystemEntity> analysisRoots;
+
+ Process? _process;
+
+ /// When not null, this is a [Completer] which completes when analysis has
+ /// finished, otherwise `null`.
+ Completer<bool>? _analysisFinished;
+
+ int _id = 0;
+
+ bool _shutdownResponseReceived = false;
+
+ final Map<String, StreamController<Map<String, dynamic>>> _streamControllers =
+ {};
+
+ final _onCrash = Completer<void>();
+
+ final Map<String, Completer<Map<String, dynamic>>> _requestCompleters = {};
+
+ final Map<String, _RequestMetadata> _requestMetadata = {};
+
+ _AnalysisServerClient(this.sdkPath, this.analysisRoots);
+
+ /// Completes when we next receive an analysis finished event (unless there's
+ /// no current analysis and we've already received a complete event, in which
+ /// case this future completes immediately).
+ Future<bool>? get analysisFinished => _analysisFinished?.future;
+
+ Stream<bool> get onAnalyzing {
+ // {"event":"server.status","params":{"analysis":{"isAnalyzing":true}}}
+ return _streamController('server.status')
+ .stream
+ .where((event) => event!['analysis'] != null)
+ .map((event) => event!['analysis']['isAnalyzing']! as bool);
+ }
+
+ /// Completes when an analysis server crash has been detected.
+ Future<void> get onCrash => _onCrash.future;
+
+ Future<int> get onExit => _process!.exitCode;
+
+ Future<bool> dispose() async {
+ return _process?.kill() ?? true;
+ }
+
+ /// Requests a completion for [file] at [offset].
+ Future<_SuggestionsData> requestCompletion(
+ String file, int offset, int maxResults) async {
+ final response = await _sendCommand('completion.getSuggestions2',
+ params: <String, dynamic>{
+ 'file': file,
+ 'offset': offset,
+ 'maxResults': maxResults,
+ });
+ final result = response['result'] as Map<String, dynamic>;
+ final metadata = _requestMetadata[response['id']]!;
+
+ final deserializeStopwatch = Stopwatch()..start();
+ final suggestionsResult = CompletionGetSuggestions2Result.fromJson(
+ ResponseDecoder(null),
+ 'result',
+ result,
+ );
+ deserializeStopwatch.stop();
+ metadata.deserializeDuration = deserializeStopwatch.elapsedMilliseconds;
+
+ return _SuggestionsData(suggestionsResult, metadata);
+ }
+
+ Future<void> shutdown({Duration timeout = const Duration(seconds: 5)}) async {
+ // Request shutdown.
+ await _sendCommand('server.shutdown').then((value) {
+ _shutdownResponseReceived = true;
+ return null;
+ }).timeout(timeout, onTimeout: () async {
+ logger.stderr('The analysis server timed out while shutting down.');
+ await dispose();
+ }).then((value) async {
+ await dispose();
+ });
+ }
+
+ Future<void> start({bool setAnalysisRoots = true}) async {
+ final process = await _startDartProcess(_sdk, [
+ _sdk.analysisServerSnapshot,
+ '--${Driver.SUPPRESS_ANALYTICS_FLAG}',
+ '--${Driver.CLIENT_ID}=completion-metrics-client',
+ '--sdk',
+ sdkPath.path,
+ ]);
+ _process = process;
+ _shutdownResponseReceived = false;
+ // This callback hookup can't throw.
+ process.exitCode.whenComplete(() {
+ _process = null;
+
+ if (!_shutdownResponseReceived) {
+ // The process exited unexpectedly. Report the crash.
+ // If `server.error` reported an error, that has been logged by
+ // `_handleServerError`.
+
+ final error = StateError('The analysis server crashed unexpectedly');
+
+ final analysisFinished = _analysisFinished;
+ if (analysisFinished != null && !analysisFinished.isCompleted) {
+ // Complete this completer in order to unstick the process.
+ analysisFinished.completeError(error);
+ }
+
+ // Complete these completers in order to unstick the process.
+ for (final completer in _requestCompleters.values) {
+ completer.completeError(error);
+ }
+
+ _onCrash.complete();
+ }
+ });
+
+ final errorStream = process.stderr
+ .transform<String>(utf8.decoder)
+ .transform<String>(const LineSplitter());
+ errorStream.listen(logger.stderr);
+
+ final inStream = process.stdout
+ .transform<String>(utf8.decoder)
+ .transform<String>(const LineSplitter());
+ inStream.listen(_handleServerResponse);
+
+ _streamController('server.error').stream.listen(_handleServerError);
+
+ _sendCommand('server.setSubscriptions', params: <String, dynamic>{
+ 'subscriptions': <String>['STATUS'],
+ });
+
+ // Reference and trim off any trailing slash, the Dart Analysis Server
+ // protocol throws an error (INVALID_FILE_PATH_FORMAT) if there is a
+ // trailing slash.
+ //
+ // The call to `absolute.resolveSymbolicLinksSync()` canonicalizes the path
+ // to be passed to the analysis server.
+ final analysisRootPaths = [
+ for (final root in analysisRoots)
+ _trimEnd(
+ root.absolute.resolveSymbolicLinksSync(), path.context.separator),
+ ];
+
+ onAnalyzing.listen((isAnalyzing) {
+ final analysisFinished = _analysisFinished;
+ if (isAnalyzing && (analysisFinished?.isCompleted ?? true)) {
+ // Start a new completer, to be completed when we receive the
+ // corresponding analysis complete event.
+ _analysisFinished = Completer();
+ } else if (!isAnalyzing &&
+ analysisFinished != null &&
+ !analysisFinished.isCompleted) {
+ analysisFinished.complete(true);
+ }
+ });
+
+ if (setAnalysisRoots) {
+ await _sendCommand('analysis.setAnalysisRoots', params: {
+ 'included': analysisRootPaths,
+ 'excluded': [],
+ });
+ }
+ }
+
+ void _handleServerError(Map<String, dynamic>? error) {
+ final err = error!;
+ // Fields are 'isFatal', 'message', and 'stackTrace'.
+ logger.stderr('Error from the analysis server: ${err['message']}');
+ if (err['stackTrace'] != null) {
+ logger.stderr(err['stackTrace'] as String);
+ }
+ }
+
+ void _handleServerResponse(String line) {
+ logger.trace('<== $line');
+
+ var responseTime = DateTime.now().millisecondsSinceEpoch;
+
+ final decodeStopwatch = Stopwatch()..start();
+ final dynamic response = json.decode(line);
+ decodeStopwatch.stop();
+ var decodeDuration = decodeStopwatch.elapsedMilliseconds;
+
+ if (response is Map<String, dynamic>) {
+ if (response['event'] != null) {
+ final event = response['event'] as String;
+ final dynamic params = response['params'];
+
+ if (params is Map<String, dynamic>) {
+ _streamController(event).add(_castStringKeyedMap(params));
+ }
+ } else if (response['id'] != null) {
+ final id = response['id'];
+ final metadata = _requestMetadata[id]!;
+ metadata.responseMilliseconds = responseTime;
+ metadata.decodeDuration = decodeDuration;
+
+ if (response['error'] != null) {
+ final error = _castStringKeyedMap(response['error']);
+ _requestCompleters
+ .remove(id)
+ ?.completeError(_RequestError.parse(error));
+ } else {
+ _requestCompleters.remove(id)?.complete(
+ //response['result'] as Map<String, dynamic>? ??
+ // <String, dynamic>{});
+ response);
+ }
+ }
+ }
+ }
+
+ Future<Map<String, dynamic>> _sendCommand(String method,
+ {Map<String, dynamic>? params}) {
+ final String id = (++_id).toString();
+ final String message = json.encode({
+ 'id': id,
+ 'method': method,
+ 'params': params,
+ });
+ _requestMetadata[id] =
+ _RequestMetadata(DateTime.now().millisecondsSinceEpoch);
+ _requestCompleters[id] = Completer();
+ _process!.stdin.writeln(message);
+ logger.trace('==> $message');
+ return _requestCompleters[id]!.future;
+ }
+
+ /// A utility method to start a Dart VM instance with the given arguments and an
+ /// optional current working directory.
+ ///
+ /// [arguments] should contain the snapshot path.
+ Future<Process> _startDartProcess(
+ _Sdk sdk,
+ List<String> arguments, {
+ String? cwd,
+ }) {
+ logger.trace('${sdk.dart} ${arguments.join(' ')}');
+ return Process.start(sdk.dart, arguments, workingDirectory: cwd);
+ }
+
+ StreamController<Map<String, dynamic>?> _streamController(String streamId) {
+ return _streamControllers.putIfAbsent(
+ streamId, () => StreamController<Map<String, dynamic>>.broadcast());
+ }
+}
+
+class _RequestError {
+ // This is copied from package:dartdev/src/analysis_server.dart.
+
+ final String code;
+
+ final String message;
+ final String stackTrace;
+ _RequestError(this.code, this.message, {required this.stackTrace});
+
+ @override
+ String toString() => '[RequestError code: $code, message: $message]';
+
+ static _RequestError parse(dynamic error) {
+ return _RequestError(
+ error['code'] as String,
+ error['message'] as String,
+ stackTrace: error['stackTrace'] as String,
+ );
+ }
+}
+
+class _RequestMetadata {
+ /// The timestamp of when a request was started, in milliseconds.
+ ///
+ /// This does not include the time it takes to encode the request into JSON.
+ final int startMilliseconds;
+
+ /// The timestamp of when a response was received, in milliseconds.
+ late final int responseMilliseconds;
+
+ /// The duration of decoding a response, in milliseconds.
+ late final int decodeDuration;
+
+ /// The duration of deserializing a response, in milliseconds.
+ late final int deserializeDuration;
+
+ _RequestMetadata(this.startMilliseconds);
+
+ /// The duration of time between sending a completion request and receiving a
+ /// completion response, not including the time to decode the response.
+ int get requestResponseDuration => responseMilliseconds - startMilliseconds;
+}
+
+/// A utility class for finding and referencing paths within the Dart SDK.
+class _Sdk {
+ // This is copied from package:dartdev/src/sdk.dart and stripped.
+
+ static final _Sdk _instance = _createSingleton();
+
+ /// Path to SDK directory.
+ final String sdkPath;
+
+ factory _Sdk() => _instance;
+
+ _Sdk._(this.sdkPath);
+
+ String get analysisServerSnapshot => path.absolute(
+ sdkPath,
+ 'bin',
+ 'snapshots',
+ 'analysis_server.dart.snapshot',
+ );
+
+ // Assume that we want to use the same Dart executable that we used to spawn
+ // DartDev. We should be able to run programs with out/ReleaseX64/dart even
+ // if the SDK isn't completely built.
+ String get dart => Platform.resolvedExecutable;
+
+ static _Sdk _createSingleton() {
+ // Find SDK path.
+
+ // The common case, and how cli_util.dart computes the Dart SDK directory,
+ // [path.dirname] called twice on Platform.resolvedExecutable. We confirm by
+ // asserting that the directory `./bin/snapshots/` exists in this directory:
+ var sdkPath =
+ path.absolute(path.dirname(path.dirname(Platform.resolvedExecutable)));
+ var snapshotsDir = path.join(sdkPath, 'bin', 'snapshots');
+ if (!Directory(snapshotsDir).existsSync()) {
+ // This is the less common case where the user is in
+ // the checked out Dart SDK, and is executing `dart` via:
+ // ./out/ReleaseX64/dart ...
+ // We confirm in a similar manner with the snapshot directory existence
+ // and then return the correct sdk path:
+ var altPath =
+ path.absolute(path.dirname(Platform.resolvedExecutable), 'dart-sdk');
+ var snapshotsDir = path.join(altPath, 'bin', 'snapshots');
+ if (Directory(snapshotsDir).existsSync()) {
+ sdkPath = altPath;
+ }
+ // If that snapshot dir does not exist either,
+ // we use the first guess anyway.
+ }
+
+ return _Sdk._(sdkPath);
+ }
+}
+
+/// A container which pairs a [CompletionGetSuggestions2Result] with the
+/// [_RequestMetadata] which is associated with the result's completion request
+/// and response.
+class _SuggestionsData {
+ final CompletionGetSuggestions2Result result;
+ final _RequestMetadata metadata;
+
+ _SuggestionsData(this.result, this.metadata);
+}
diff --git a/tools/VERSION b/tools/VERSION
index a5e92bf..bb151a6 100644
--- a/tools/VERSION
+++ b/tools/VERSION
@@ -27,5 +27,5 @@
MAJOR 2
MINOR 18
PATCH 0
-PRERELEASE 83
+PRERELEASE 84
PRERELEASE_PATCH 0
\ No newline at end of file
