| // Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'dart:convert'; |
| import 'dart:io' as io; |
| import 'dart:math' as math; |
| |
| import 'package:_fe_analyzer_shared/src/base/syntactic_entity.dart'; |
| import 'package:analysis_server/src/domains/completion/available_suggestions.dart'; |
| import 'package:analysis_server/src/protocol/protocol_internal.dart'; |
| import 'package:analysis_server/src/protocol_server.dart' as protocol; |
| import 'package:analysis_server/src/services/completion/completion_core.dart'; |
| import 'package:analysis_server/src/services/completion/completion_performance.dart'; |
| import 'package:analysis_server/src/services/completion/dart/completion_manager.dart'; |
| import 'package:analysis_server/src/services/completion/dart/feature_computer.dart'; |
| import 'package:analysis_server/src/services/completion/dart/probability_range.dart'; |
| import 'package:analysis_server/src/services/completion/dart/relevance_tables.g.dart'; |
| import 'package:analysis_server/src/services/completion/dart/suggestion_builder.dart'; |
| import 'package:analysis_server/src/services/completion/dart/utilities.dart'; |
| import 'package:analysis_server/src/status/pages.dart'; |
| import 'package:analyzer/dart/analysis/context_root.dart'; |
| import 'package:analyzer/dart/analysis/results.dart'; |
| import 'package:analyzer/dart/ast/ast.dart'; |
| import 'package:analyzer/dart/ast/token.dart'; |
| import 'package:analyzer/dart/element/element.dart' |
| show |
| ClassElement, |
| ClassMemberElement, |
| CompilationUnitElement, |
| Element, |
| ExecutableElement, |
| ExtensionElement, |
| FieldElement, |
| FunctionElement, |
| LocalVariableElement, |
| ParameterElement, |
| PrefixElement, |
| TypeParameterElement, |
| VariableElement; |
| import 'package:analyzer/diagnostic/diagnostic.dart'; |
| import 'package:analyzer/error/error.dart' as err; |
| import 'package:analyzer/file_system/file_system.dart'; |
| import 'package:analyzer/file_system/overlay_file_system.dart'; |
| import 'package:analyzer/file_system/physical_file_system.dart'; |
| import 'package:analyzer/src/dart/analysis/analysis_context_collection.dart'; |
| import 'package:analyzer/src/dart/analysis/byte_store.dart'; |
| import 'package:analyzer/src/dartdoc/dartdoc_directive_info.dart'; |
| import 'package:analyzer/src/services/available_declarations.dart'; |
| import 'package:analyzer/src/util/file_paths.dart' as file_paths; |
| import 'package:analyzer/src/util/performance/operation_performance.dart'; |
| import 'package:analyzer_plugin/src/utilities/completion/optype.dart'; |
| import 'package:args/args.dart'; |
| import 'package:meta/meta.dart'; |
| |
| import 'metrics_util.dart'; |
| import 'output_utilities.dart'; |
| import 'relevance_table_generator.dart'; |
| import 'visitors.dart'; |
| |
| Future<void> main(List<String> args) async { |
| var parser = createArgParser(); |
| var result = parser.parse(args); |
| |
| if (!validArguments(parser, result)) { |
| return io.exit(1); |
| } |
| |
| var options = CompletionMetricsOptions(result); |
| var provider = PhysicalResourceProvider.INSTANCE; |
| if (result.wasParsed('reduceDir')) { |
| var targetMetrics = <CompletionMetrics>[]; |
| var dir = provider.getFolder(result['reduceDir']); |
| var computer = CompletionMetricsComputer('', options); |
| for (var child in dir.getChildren()) { |
| if (child is File) { |
| var metricsList = |
| (json.decode(child.readAsStringSync()) as List<dynamic>) |
| .map((map) => |
| CompletionMetrics.fromJson(map as Map<String, dynamic>)) |
| .toList(); |
| if (targetMetrics.isEmpty) { |
| targetMetrics.addAll(metricsList); |
| } else if (targetMetrics.length != metricsList.length) { |
| throw StateError('metrics lengths differ'); |
| } else { |
| for (var i = 0; i < targetMetrics.length; i++) { |
| targetMetrics[i].addData(metricsList[i]); |
| } |
| } |
| } |
| } |
| computer.targetMetrics.addAll(targetMetrics); |
| computer.printResults(); |
| return; |
| } |
| |
| var rootPath = result.rest[0]; |
| print('Analyzing root: "$rootPath"'); |
| var stopwatch = Stopwatch()..start(); |
| var computer = CompletionMetricsComputer(rootPath, options); |
| var code = await computer.computeMetrics(); |
| stopwatch.stop(); |
| |
| var duration = Duration(milliseconds: stopwatch.elapsedMilliseconds); |
| print(''); |
| print('Metrics computed in $duration'); |
| |
| File uniqueDataFile() { |
| var dataDir = result['mapDir']; |
| var baseFileName = provider.pathContext.basename(rootPath); |
| var index = 1; |
| while (index < 10000) { |
| var suffix = (index++).toString(); |
| suffix = '0000'.substring(suffix.length) + suffix + '.json'; |
| var fileName = baseFileName + suffix; |
| var filePath = provider.pathContext.join(dataDir, fileName); |
| var file = provider.getFile(filePath); |
| if (!file.exists) { |
| return file; |
| } |
| } |
| |
| /// If there are more than 10000 directories with the same name, just |
| /// overwrite a previously generated file. |
| var fileName = baseFileName + '9999'; |
| var filePath = provider.pathContext.join(dataDir, fileName); |
| return provider.getFile(filePath); |
| } |
| |
| if (result.wasParsed('mapDir')) { |
| var dataFile = uniqueDataFile(); |
| var map = |
| computer.targetMetrics.map((metrics) => metrics.toJson()).toList(); |
| dataFile.writeAsStringSync(json.encode(map)); |
| } else { |
| computer.printResults(); |
| } |
| return io.exit(code); |
| } |
| |
| /// A [Counter] to track the performance of each of the completion strategies |
| /// that are being compared. |
| Counter rankComparison = Counter('relevance rank comparison'); |
| |
| /// Create a parser that can be used to parse the command-line arguments. |
| ArgParser createArgParser() { |
| return ArgParser() |
| ..addFlag(CompletionMetricsOptions.AVAILABLE_SUGGESTIONS, |
| abbr: 'a', |
| help: |
| 'Use the available suggestions feature in the Analysis Server when ' |
| 'computing the set of code completions. With this feature enabled, ' |
| 'completion will match the support in the Dart Plugin for ' |
| 'IntelliJ, without this enabled the completion support matches the ' |
| 'support in LSP.', |
| defaultsTo: false, |
| negatable: false) |
| ..addOption( |
| 'help', |
| abbr: 'h', |
| help: 'Print this help message.', |
| ) |
| ..addOption(CompletionMetricsOptions.OVERLAY, |
| allowed: [ |
| CompletionMetricsOptions.OVERLAY_NONE, |
| CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN, |
| CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE |
| ], |
| defaultsTo: CompletionMetricsOptions.OVERLAY_NONE, |
| help: |
| 'Before attempting a completion at the location of each token, the ' |
| 'token can be removed, or the rest of the file can be removed to ' |
| 'test code completion with diverse methods. The default mode is to ' |
| 'complete at the start of the token without modifying the file.') |
| ..addFlag(CompletionMetricsOptions.PRINT_MISSED_COMPLETION_DETAILS, |
| defaultsTo: false, |
| help: |
| 'Print detailed information every time a completion request fails ' |
| 'to produce a suggestions matching the expected suggestion.', |
| negatable: false) |
| ..addFlag(CompletionMetricsOptions.PRINT_MISSED_COMPLETION_SUMMARY, |
| defaultsTo: false, |
| help: 'Print summary information about the times that a completion ' |
| 'request failed to produce a suggestions matching the expected ' |
| 'suggestion.', |
| negatable: false) |
| ..addFlag(CompletionMetricsOptions.PRINT_MISSING_INFORMATION, |
| defaultsTo: false, |
| help: 'Print information about places where no completion location was ' |
| 'computed and about information that is missing in the completion ' |
| 'tables.', |
| negatable: false) |
| ..addFlag(CompletionMetricsOptions.PRINT_MRR_BY_LOCATION, |
| defaultsTo: false, |
| help: |
| 'Print information about the mrr score achieved at each completion ' |
| 'location. This can help focus efforts to improve the overall ' |
| 'score by pointing out the locations that are causing the biggest ' |
| 'impact.', |
| negatable: false) |
| ..addFlag(CompletionMetricsOptions.PRINT_SLOWEST_RESULTS, |
| defaultsTo: false, |
| help: 'Print information about the completion requests that were the ' |
| 'slowest to return suggestions.', |
| negatable: false) |
| ..addFlag(CompletionMetricsOptions.PRINT_WORST_RESULTS, |
| defaultsTo: false, |
| help: 'Print information about the completion requests that had the ' |
| 'worst mrr scores.', |
| negatable: false) |
| ..addOption( |
| 'mapDir', |
| help: 'The absolute path of the directory to which the completion ' |
| 'metrics data will be written. Using this option will prevent the ' |
| 'completion results from being written in a textual form.', |
| ) |
| ..addOption( |
| 'reduceDir', |
| help: 'The absolute path of the directory from which the completion ' |
| 'metrics data will be read.', |
| ); |
| } |
| |
| /// Print usage information for this tool. |
| void printUsage(ArgParser parser, {String error}) { |
| if (error != null) { |
| print(error); |
| print(''); |
| } |
| print('usage: dart completion_metrics.dart [options] packagePath'); |
| print(''); |
| print('Compute code completion health metrics.'); |
| print(''); |
| print(parser.usage); |
| } |
| |
| /// Return `true` if the command-line arguments (represented by the [result] and |
| /// parsed by the [parser]) are valid. |
| bool validArguments(ArgParser parser, ArgResults result) { |
| if (result.wasParsed('help')) { |
| printUsage(parser); |
| return false; |
| } else if (result.wasParsed('reduceDir')) { |
| return validateDir(parser, result['reduceDir']); |
| } else if (result.rest.length != 1) { |
| printUsage(parser, error: 'No package path specified.'); |
| return false; |
| } |
| if (result.wasParsed('mapDir')) { |
| return validateDir(parser, result['mapDir']); |
| } |
| return validateDir(parser, result.rest[0]); |
| } |
| |
| /// An indication of the group in which the completion falls for the purposes of |
| /// subdividing the results. |
| enum CompletionGroup { |
| classElement, |
| constructorElement, |
| enumElement, |
| extensionElement, |
| |
| /// An instance member of a class, enum, mixin or extension. |
| instanceMember, |
| |
| labelElement, |
| localFunctionElement, |
| localVariableElement, |
| mixinElement, |
| parameterElement, |
| prefixElement, |
| |
| /// A static member of a class, enum, mixin or extension. |
| staticMember, |
| |
| topLevelMember, |
| typeParameterElement, |
| |
| // Groups for keywords. |
| |
| keywordDynamic, |
| keywordVoid, |
| |
| /// Anything that doesn't fit in one of the other groups. |
| unknown, |
| } |
| |
| /// A wrapper for the collection of [Counter] and [MeanReciprocalRankComputer] |
| /// objects for a run of [CompletionMetricsComputer]. |
| class CompletionMetrics { |
| /// The maximum number of slowest results to collect. |
| static const maxSlowestResults = 5; |
| |
| /// The maximum number of worst results to collect. |
| static const maxWorstResults = 5; |
| |
| /// The name associated with this set of metrics. |
| final String name; |
| |
| /// The function to be executed when this metrics collector is enabled. |
| final void Function() enableFunction; |
| |
| /// The function to be executed when this metrics collector is disabled. |
| final void Function() disableFunction; |
| |
| final Counter completionCounter = Counter('all completions'); |
| |
| final Counter completionMissedTokenCounter = |
| Counter('unsuccessful completion token counter'); |
| |
| final Counter completionKindCounter = |
| Counter('unsuccessful completion kind counter'); |
| |
| final Counter completionElementKindCounter = |
| Counter('unsuccessful completion element kind counter'); |
| |
| final ArithmeticMeanComputer meanCompletionMS = |
| ArithmeticMeanComputer('ms per completion'); |
| |
| final MeanReciprocalRankComputer mrrComputer = |
| MeanReciprocalRankComputer('all completions'); |
| |
| final MeanReciprocalRankComputer successfulMrrComputer = |
| MeanReciprocalRankComputer('successful completions'); |
| |
| /// A table mapping completion groups to the mrr computer used to track the |
| /// quality of suggestions for those groups. |
| final Map<CompletionGroup, MeanReciprocalRankComputer> groupMrrComputers = {}; |
| |
| /// A table mapping locations to the mrr computer used to track the quality of |
| /// suggestions for those locations. |
| final Map<String, MeanReciprocalRankComputer> locationMrrComputers = {}; |
| |
| final ArithmeticMeanComputer charsBeforeTop = |
| ArithmeticMeanComputer('chars_before_top'); |
| |
| final ArithmeticMeanComputer charsBeforeTopFive = |
| ArithmeticMeanComputer('chars_before_top_five'); |
| |
| final ArithmeticMeanComputer insertionLengthTheoretical = |
| ArithmeticMeanComputer('insertion_length_theoretical'); |
| |
| /// The places in which a completion location was requested when none was |
| /// available. |
| final Set<String> missingCompletionLocations = {}; |
| |
| /// The completion locations for which no relevance table was available. |
| final Set<String> missingCompletionLocationTables = {}; |
| |
| final Map<CompletionGroup, List<CompletionResult>> slowestResults = {}; |
| |
| final Map<CompletionGroup, List<CompletionResult>> worstResults = {}; |
| |
| CompletionMetrics(this.name, {this.enableFunction, this.disableFunction}); |
| |
| /// Return an instance extracted from the decoded JSON [map]. |
| factory CompletionMetrics.fromJson(Map<String, dynamic> map) { |
| var metrics = CompletionMetrics(map['name'] as String); |
| metrics.completionCounter |
| .fromJson(map['completionCounter'] as Map<String, dynamic>); |
| metrics.completionMissedTokenCounter |
| .fromJson(map['completionMissedTokenCounter'] as Map<String, dynamic>); |
| metrics.completionKindCounter |
| .fromJson(map['completionKindCounter'] as Map<String, dynamic>); |
| metrics.completionElementKindCounter |
| .fromJson(map['completionElementKindCounter'] as Map<String, dynamic>); |
| metrics.meanCompletionMS |
| .fromJson(map['meanCompletionMS'] as Map<String, dynamic>); |
| metrics.mrrComputer.fromJson(map['mrrComputer'] as Map<String, dynamic>); |
| metrics.successfulMrrComputer |
| .fromJson(map['successfulMrrComputer'] as Map<String, dynamic>); |
| for (var entry |
| in (map['groupMrrComputers'] as Map<String, dynamic>).entries) { |
| var group = CompletionGroup.values[int.parse(entry.key)]; |
| metrics.groupMrrComputers[group] = MeanReciprocalRankComputer(group.name) |
| ..fromJson(entry.value); |
| } |
| for (var entry |
| in (map['locationMrrComputers'] as Map<String, dynamic>).entries) { |
| var location = entry.key; |
| metrics.locationMrrComputers[location] = |
| MeanReciprocalRankComputer(location)..fromJson(entry.value); |
| } |
| metrics.charsBeforeTop |
| .fromJson(map['charsBeforeTop'] as Map<String, dynamic>); |
| metrics.charsBeforeTopFive |
| .fromJson(map['charsBeforeTopFive'] as Map<String, dynamic>); |
| metrics.insertionLengthTheoretical |
| .fromJson(map['insertionLengthTheoretical'] as Map<String, dynamic>); |
| for (var element in map['missingCompletionLocations'] as List<dynamic>) { |
| metrics.missingCompletionLocations.add(element); |
| } |
| for (var element |
| in map['missingCompletionLocationTables'] as List<dynamic>) { |
| metrics.missingCompletionLocationTables.add(element); |
| } |
| for (var entry in (map['slowestResults'] as Map<String, dynamic>).entries) { |
| var group = CompletionGroup.values[int.parse(entry.key)]; |
| var results = (entry.value as List<dynamic>) |
| .map((map) => CompletionResult.fromJson(map as Map<String, dynamic>)) |
| .toList(); |
| metrics.slowestResults[group] = results; |
| } |
| for (var entry in (map['worstResults'] as Map<String, dynamic>).entries) { |
| var group = CompletionGroup.values[int.parse(entry.key)]; |
| var results = (entry.value as List<dynamic>) |
| .map((map) => CompletionResult.fromJson(map as Map<String, dynamic>)) |
| .toList(); |
| metrics.worstResults[group] = results; |
| } |
| return metrics; |
| } |
| |
| /// Add the data from the given [metrics] to this metrics. |
| void addData(CompletionMetrics metrics) { |
| completionCounter.addData(metrics.completionCounter); |
| completionMissedTokenCounter.addData(metrics.completionMissedTokenCounter); |
| completionKindCounter.addData(metrics.completionKindCounter); |
| completionElementKindCounter.addData(metrics.completionElementKindCounter); |
| meanCompletionMS.addData(metrics.meanCompletionMS); |
| mrrComputer.addData(metrics.mrrComputer); |
| successfulMrrComputer.addData(metrics.successfulMrrComputer); |
| for (var entry in metrics.groupMrrComputers.entries) { |
| var group = entry.key; |
| groupMrrComputers |
| .putIfAbsent(group, () => MeanReciprocalRankComputer(group.name)) |
| .addData(entry.value); |
| } |
| for (var entry in metrics.locationMrrComputers.entries) { |
| var location = entry.key; |
| locationMrrComputers |
| .putIfAbsent(location, () => MeanReciprocalRankComputer(location)) |
| .addData(entry.value); |
| } |
| charsBeforeTop.addData(metrics.charsBeforeTop); |
| charsBeforeTopFive.addData(metrics.charsBeforeTopFive); |
| insertionLengthTheoretical.addData(metrics.insertionLengthTheoretical); |
| missingCompletionLocations.addAll(metrics.missingCompletionLocations); |
| missingCompletionLocationTables |
| .addAll(metrics.missingCompletionLocationTables); |
| for (var resultList in metrics.slowestResults.values) { |
| for (var result in resultList) { |
| _recordSlowestResult(result); |
| } |
| } |
| for (var resultList in metrics.worstResults.values) { |
| for (var result in resultList) { |
| _recordWorstResult(result); |
| } |
| } |
| } |
| |
| /// Perform any operations required in order to revert computing the kind of |
| /// completions represented by this metrics collector. |
| void disable() { |
| if (disableFunction != null) { |
| disableFunction(); |
| } |
| } |
| |
| /// Perform any initialization required in order to compute the kind of |
| /// completions represented by this metrics collector. |
| void enable() { |
| if (enableFunction != null) { |
| enableFunction(); |
| } |
| } |
| |
| /// Record the completion [result]. This method handles the worst ranked items |
| /// as well as the longest sets of results to compute. |
| void recordCompletionResult( |
| CompletionResult result, MetricsSuggestionListener listener) { |
| _recordTime(result); |
| _recordMrr(result); |
| _recordWorstResult(result); |
| _recordSlowestResult(result); |
| _recordMissingInformation(listener); |
| } |
| |
| Map<String, dynamic> toJson() { |
| return { |
| 'name': name, |
| 'completionCounter': completionCounter.toJson(), |
| 'completionMissedTokenCounter': completionMissedTokenCounter.toJson(), |
| 'completionKindCounter': completionKindCounter.toJson(), |
| 'completionElementKindCounter': completionElementKindCounter.toJson(), |
| 'meanCompletionMS': meanCompletionMS.toJson(), |
| 'mrrComputer': mrrComputer.toJson(), |
| 'successfulMrrComputer': successfulMrrComputer.toJson(), |
| 'groupMrrComputers': groupMrrComputers |
| .map((key, value) => MapEntry(key.index.toString(), value.toJson())), |
| 'locationMrrComputers': locationMrrComputers |
| .map((key, value) => MapEntry(key, value.toJson())), |
| 'charsBeforeTop': charsBeforeTop.toJson(), |
| 'charsBeforeTopFive': charsBeforeTopFive.toJson(), |
| 'insertionLengthTheoretical': insertionLengthTheoretical.toJson(), |
| 'missingCompletionLocations': missingCompletionLocations.toList(), |
| 'missingCompletionLocationTables': |
| missingCompletionLocationTables.toList(), |
| 'slowestResults': slowestResults.map((key, value) => MapEntry( |
| key.index.toString(), |
| value.map((result) => result.toJson()).toList())), |
| 'worstResults': worstResults.map((key, value) => MapEntry( |
| key.index.toString(), |
| value.map((result) => result.toJson()).toList())), |
| }; |
| } |
| |
| /// If the completion location was requested but missing when computing the |
| /// [result], then record where that happened. |
| void _recordMissingInformation(MetricsSuggestionListener listener) { |
| var location = listener?.missingCompletionLocation; |
| if (location != null) { |
| missingCompletionLocations.add(location); |
| } else { |
| location = listener?.missingCompletionLocationTable; |
| if (location != null) { |
| missingCompletionLocationTables.add(location); |
| } |
| } |
| } |
| |
| /// Record the MRR for the [result]. |
| void _recordMrr(CompletionResult result) { |
| var rank = result.place.rank; |
| // Record globally. |
| successfulMrrComputer.addRank(rank); |
| // Record by group. |
| var group = result.group; |
| groupMrrComputers |
| .putIfAbsent(group, () => MeanReciprocalRankComputer(group.name)) |
| .addRank(rank); |
| // Record by completion location. |
| var location = result.completionLocation; |
| if (location != null) { |
| var computer = locationMrrComputers.putIfAbsent( |
| location, () => MeanReciprocalRankComputer(location)); |
| computer.addRank(rank); |
| } |
| } |
| |
| /// If the [result] is took longer than any previously recorded results, |
| /// record it. |
| void _recordSlowestResult(CompletionResult result) { |
| var results = slowestResults.putIfAbsent(result.group, () => []); |
| if (results.length >= maxSlowestResults) { |
| if (result.elapsedMS <= results.last.elapsedMS) { |
| return; |
| } |
| results.removeLast(); |
| } |
| results.add(result); |
| results.sort((first, second) => second.elapsedMS - first.elapsedMS); |
| } |
| |
| /// Record this elapsed ms count for the average ms count. |
| void _recordTime(CompletionResult result) { |
| meanCompletionMS.addValue(result.elapsedMS); |
| } |
| |
| /// If the [result] is worse than any previously recorded results, record it. |
| void _recordWorstResult(CompletionResult result) { |
| var results = worstResults.putIfAbsent(result.group, () => []); |
| if (results.length >= maxWorstResults) { |
| if (result.place.rank <= results.last.place.rank) { |
| return; |
| } |
| results.removeLast(); |
| } |
| results.add(result); |
| results.sort((first, second) => second.place.rank - first.place.rank); |
| } |
| } |
| |
| /// This is the main metrics computer class for code completions. After the |
| /// object is constructed, [computeCompletionMetrics] is executed to do analysis |
| /// and print a summary of the metrics gathered from the completion tests. |
| class CompletionMetricsComputer { |
| final String rootPath; |
| |
| final CompletionMetricsOptions options; |
| |
| ResolvedUnitResult _resolvedUnitResult; |
| |
| /// The int to be returned from the [computeMetrics] call. |
| int resultCode; |
| |
| /// A list of the metrics to be computed. |
| final List<CompletionMetrics> targetMetrics = []; |
| |
| final OverlayResourceProvider _provider = |
| OverlayResourceProvider(PhysicalResourceProvider.INSTANCE); |
| |
| int overlayModificationStamp = 0; |
| |
| CompletionMetricsComputer(this.rootPath, this.options); |
| |
| /// Compare the metrics when each feature is used in isolation. |
| void compareIndividualFeatures() { |
| var featureNames = FeatureComputer.featureNames; |
| var featureCount = featureNames.length; |
| for (var i = 0; i < featureCount; i++) { |
| var weights = List.filled(featureCount, 0.00); |
| weights[i] = 1.00; |
| targetMetrics.add(CompletionMetrics(featureNames[i], enableFunction: () { |
| FeatureComputer.featureWeights = weights; |
| }, disableFunction: () { |
| FeatureComputer.featureWeights = FeatureComputer.defaultFeatureWeights; |
| })); |
| } |
| } |
| |
| /// Compare the relevance [tables] to the default relevance tables. |
| void compareRelevanceTables(List<RelevanceTables> tables) { |
| assert(tables.isNotEmpty); |
| for (var tablePair in tables) { |
| targetMetrics.add(CompletionMetrics(tablePair.name, enableFunction: () { |
| elementKindRelevance = tablePair.elementKindRelevance; |
| keywordRelevance = tablePair.keywordRelevance; |
| }, disableFunction: () { |
| elementKindRelevance = defaultElementKindRelevance; |
| keywordRelevance = defaultKeywordRelevance; |
| })); |
| } |
| } |
| |
| Future<int> computeMetrics() async { |
| resultCode = 0; |
| // To compare two or more changes to completions, add a `CompletionMetrics` |
| // object with enable and disable functions to the list of `targetMetrics`. |
| targetMetrics.add(CompletionMetrics('shipping', |
| enableFunction: null, disableFunction: null)); |
| |
| // To compare two or more relevance tables, uncomment the line below and |
| // add the `RelevanceTables` to the list. The default relevance tables |
| // should not be included in the list. |
| // compareRelevanceTables([]); |
| |
| // To compare the relative benefit from each of the features, uncomment the |
| // line below. |
| // compareIndividualFeatures(); |
| |
| final collection = AnalysisContextCollectionImpl( |
| includedPaths: [rootPath], |
| resourceProvider: PhysicalResourceProvider.INSTANCE, |
| ); |
| for (var context in collection.contexts) { |
| await _computeInContext(context.contextRoot); |
| } |
| return resultCode; |
| } |
| |
| int forEachExpectedCompletion( |
| CompletionRequestImpl request, |
| MetricsSuggestionListener listener, |
| ExpectedCompletion expectedCompletion, |
| String completionLocation, |
| List<protocol.CompletionSuggestion> suggestions, |
| CompletionMetrics metrics, |
| int elapsedMS) { |
| assert(suggestions != null); |
| |
| var place = placementInSuggestionList(suggestions, expectedCompletion); |
| |
| metrics.mrrComputer.addRank(place.rank); |
| |
| if (place.denominator != 0) { |
| metrics.completionCounter.count('successful'); |
| |
| var rank = place.rank; |
| var suggestion = suggestions[rank - 1]; |
| var actualSuggestion = |
| SuggestionData(suggestion, listener.featureMap[suggestion]); |
| List<SuggestionData> topSuggestions; |
| Map<int, int> precedingRelevanceCounts; |
| if (options.printWorstResults) { |
| topSuggestions = suggestions |
| .sublist(0, math.min(10, suggestions.length)) |
| .map((suggestion) => |
| SuggestionData(suggestion, listener.featureMap[suggestion])) |
| .toList(); |
| precedingRelevanceCounts = <int, int>{}; |
| for (var i = 0; i < rank - 1; i++) { |
| var relevance = suggestions[i].relevance; |
| precedingRelevanceCounts[relevance] = |
| (precedingRelevanceCounts[relevance] ?? 0) + 1; |
| } |
| } |
| metrics.recordCompletionResult( |
| CompletionResult( |
| place, |
| request, |
| actualSuggestion, |
| topSuggestions, |
| precedingRelevanceCounts, |
| expectedCompletion, |
| completionLocation, |
| elapsedMS), |
| listener); |
| |
| var charsBeforeTop = |
| _computeCharsBeforeTop(expectedCompletion, suggestions); |
| metrics.charsBeforeTop.addValue(charsBeforeTop); |
| metrics.charsBeforeTopFive.addValue( |
| _computeCharsBeforeTop(expectedCompletion, suggestions, minRank: 5)); |
| metrics.insertionLengthTheoretical |
| .addValue(expectedCompletion.completion.length - charsBeforeTop); |
| |
| return place.rank; |
| } else { |
| metrics.completionCounter.count('unsuccessful'); |
| |
| metrics.completionMissedTokenCounter.count(expectedCompletion.completion); |
| metrics.completionKindCounter.count(expectedCompletion.kind.toString()); |
| metrics.completionElementKindCounter |
| .count(expectedCompletion.elementKind.toString()); |
| |
| if (options.printMissedCompletionDetails) { |
| protocol.CompletionSuggestion closeMatchSuggestion; |
| for (var suggestion in suggestions) { |
| if (suggestion.completion == expectedCompletion.completion) { |
| closeMatchSuggestion = suggestion; |
| } |
| } |
| |
| print('missing completion (${metrics.name}):'); |
| print('$expectedCompletion'); |
| if (closeMatchSuggestion != null) { |
| print(' close matching completion that was in the list:'); |
| print(' $closeMatchSuggestion'); |
| } |
| print(''); |
| } |
| |
| return -1; |
| } |
| } |
| |
| void printComparisons() { |
| printHeading(1, 'Comparison of experiments'); |
| printMrrComparison(); |
| printCounter(rankComparison); |
| printOtherMetrics(); |
| printCompletionCounts(); |
| } |
| |
| void printCompletionCounts() { |
| String toString(int count, int totalCount) { |
| return '$count (${printPercentage(count / totalCount, 2)})'; |
| } |
| |
| var counters = targetMetrics.map((metrics) => metrics.completionCounter); |
| var table = [ |
| ['', for (var metrics in targetMetrics) metrics.name], |
| ['total', for (var counter in counters) counter.totalCount.toString()], |
| [ |
| 'successful', |
| for (var counter in counters) |
| toString(counter.getCountOf('successful'), counter.totalCount) |
| ], |
| [ |
| 'unsuccessful', |
| for (var counter in counters) |
| toString(counter.getCountOf('unsuccessful'), counter.totalCount) |
| ], |
| ]; |
| rightJustifyColumns(table, range(1, table[0].length)); |
| |
| printHeading(2, 'Comparison of completion counts'); |
| printTable(table); |
| } |
| |
| void printCounter(Counter counter) { |
| var name = counter.name; |
| var total = counter.totalCount; |
| printHeading(2, "Counts for '$name' (total = $total)"); |
| counter.printCounterValues(); |
| } |
| |
| void printHeading(int level, String heading) { |
| var prefix = '#' * level; |
| print('$prefix $heading'); |
| print(''); |
| } |
| |
| void printMetrics(CompletionMetrics metrics) { |
| printHeading(1, 'Completion metrics for ${metrics.name}'); |
| if (options.printMissedCompletionSummary) { |
| printCounter(metrics.completionMissedTokenCounter); |
| printCounter(metrics.completionKindCounter); |
| printCounter(metrics.completionElementKindCounter); |
| } |
| |
| List<String> toRow(MeanReciprocalRankComputer computer) { |
| return [ |
| computer.name, |
| computer.mrr.toStringAsFixed(6), |
| (1 / computer.mrr).toStringAsFixed(3), |
| computer.mrr_5.toStringAsFixed(6), |
| (1 / computer.mrr_5).toStringAsFixed(3), |
| computer.count.toString(), |
| ]; |
| } |
| |
| var groups = metrics.groupMrrComputers.keys.toList(); |
| groups.sort((first, second) => first.name.compareTo(second.name)); |
| var table = [ |
| ['', 'mrr', 'inverse mrr', 'mrr_5', 'inverse mrr_5', 'count'], |
| toRow(metrics.mrrComputer), |
| toRow(metrics.successfulMrrComputer), |
| ['', '', '', '', '', ''], |
| for (var group in groups) toRow(metrics.groupMrrComputers[group]), |
| ]; |
| rightJustifyColumns(table, [2, 4, 5]); |
| |
| printHeading(2, 'Mean Reciprocal Rank'); |
| printTable(table); |
| |
| if (options.printMrrByLocation) { |
| var lines = <LocationTableLine>[]; |
| for (var entry in metrics.locationMrrComputers.entries) { |
| var count = entry.value.count; |
| var mrr = 1 / entry.value.mrr; |
| var mrr_5 = 1 / entry.value.mrr_5; |
| var product = count * mrr; |
| lines.add(LocationTableLine( |
| label: entry.key, |
| product: product, |
| count: count, |
| mrr: mrr, |
| mrr_5: mrr_5)); |
| } |
| lines.sort((first, second) => second.product.compareTo(first.product)); |
| var table = <List<String>>[]; |
| table.add(['Location', 'Product', 'Count', 'Mrr', 'Mrr_5']); |
| for (var line in lines) { |
| var location = line.label; |
| var product = line.product.truncate().toString(); |
| var count = line.count.toString(); |
| var mrr = line.mrr.toStringAsFixed(3); |
| var mrr_5 = line.mrr_5.toStringAsFixed(3); |
| table.add([location, product, count, mrr, mrr_5]); |
| } |
| printTable(table); |
| } |
| } |
| |
| void printMissingInformation(CompletionMetrics metrics) { |
| var locations = metrics.missingCompletionLocations; |
| if (locations.isNotEmpty) { |
| print(''); |
| printHeading(2, 'Missing completion location in the following places'); |
| for (var location in locations.toList()..sort()) { |
| print('- $location'); |
| } |
| } |
| |
| var tables = metrics.missingCompletionLocationTables; |
| if (tables.isNotEmpty) { |
| print(''); |
| printHeading(2, 'Missing tables for the following completion locations'); |
| for (var table in tables.toList()..sort()) { |
| print('- $table'); |
| } |
| } |
| } |
| |
| void printMrrComparison() { |
| List<String> toRow(Iterable<MeanReciprocalRankComputer> sources) { |
| var computers = sources.toList(); |
| var baseComputer = computers.first; |
| var row = [baseComputer.name]; |
| var baseInverseMrr = 1 / baseComputer.mrr; |
| row.add(baseInverseMrr.toStringAsFixed(3)); |
| for (var i = 1; i < computers.length; i++) { |
| var inverseMrr = 1 / computers[i].mrr; |
| var delta = inverseMrr - baseInverseMrr; |
| row.add('|'); |
| row.add(inverseMrr.toStringAsFixed(3)); |
| row.add(delta.toStringAsFixed(3)); |
| } |
| return row; |
| } |
| |
| var columnHeaders = [' ', targetMetrics[0].name]; |
| for (var i = 1; i < targetMetrics.length; i++) { |
| columnHeaders.add('|'); |
| columnHeaders.add('${targetMetrics[i].name}'); |
| columnHeaders.add('delta'); |
| } |
| var blankRow = [for (int i = 0; i < columnHeaders.length; i++) '']; |
| var table = [ |
| columnHeaders, |
| toRow(targetMetrics.map((metrics) => metrics.mrrComputer)), |
| toRow(targetMetrics.map((metrics) => metrics.successfulMrrComputer)), |
| blankRow, |
| ]; |
| var elementKinds = targetMetrics |
| .expand((metrics) => metrics.groupMrrComputers.keys) |
| .toSet() |
| .toList(); |
| elementKinds.sort((first, second) => first.name.compareTo(second.name)); |
| for (var kind in elementKinds) { |
| table.add(toRow( |
| targetMetrics.map((metrics) => metrics.groupMrrComputers[kind]))); |
| } |
| if (options.printMrrByLocation) { |
| table.add(blankRow); |
| var locations = targetMetrics |
| .expand((metrics) => metrics.locationMrrComputers.keys) |
| .toSet() |
| .toList(); |
| locations.sort(); |
| for (var location in locations) { |
| table.add(toRow(targetMetrics |
| .map((metrics) => metrics.locationMrrComputers[location]))); |
| } |
| } |
| rightJustifyColumns(table, range(1, table[0].length)); |
| |
| printHeading(2, 'Comparison of inverse mean reciprocal ranks'); |
| print('A lower value is better, so a negative delta is good.'); |
| print(''); |
| printTable(table); |
| } |
| |
| void printOtherMetrics() { |
| List<String> toRow(Iterable<ArithmeticMeanComputer> sources) { |
| var computers = sources.toList(); |
| var row = [computers.first.name]; |
| for (var computer in computers) { |
| row.add(computer.mean.toStringAsFixed(6)); |
| } |
| return row; |
| } |
| |
| var table = [ |
| ['', for (var metrics in targetMetrics) metrics.name], |
| toRow(targetMetrics.map((metrics) => metrics.meanCompletionMS)), |
| toRow(targetMetrics.map((metrics) => metrics.charsBeforeTop)), |
| toRow(targetMetrics.map((metrics) => metrics.charsBeforeTopFive)), |
| toRow(targetMetrics.map((metrics) => metrics.insertionLengthTheoretical)), |
| ]; |
| rightJustifyColumns(table, range(1, table[0].length)); |
| |
| printHeading(2, 'Comparison of other metrics'); |
| printTable(table); |
| } |
| |
| void printResults() { |
| if (targetMetrics.length > 1) { |
| print(''); |
| printComparisons(); |
| } |
| var needsBlankLine = false; |
| for (var metrics in targetMetrics) { |
| if (needsBlankLine) { |
| print(''); |
| } else { |
| needsBlankLine = true; |
| } |
| printMetrics(metrics); |
| |
| if (options.printMissingInformation) { |
| printMissingInformation(metrics); |
| } |
| if (options.printSlowestResults) { |
| printSlowestResults(metrics); |
| } |
| if (options.printWorstResults) { |
| printWorstResults(metrics); |
| } |
| } |
| } |
| |
| void printSlowestResults(CompletionMetrics metrics) { |
| var slowestResults = metrics.slowestResults; |
| var groups = slowestResults.keys.toList(); |
| groups.sort((first, second) => first.name.compareTo(second.name)); |
| print(''); |
| printHeading(2, 'The slowest completion results to compute'); |
| for (var group in groups) { |
| _printSlowestResults('In ${group.name}', slowestResults[group]); |
| } |
| } |
| |
| void printWorstResults(CompletionMetrics metrics) { |
| var worstResults = metrics.worstResults; |
| var groups = worstResults.keys.toList(); |
| groups.sort((first, second) => first.name.compareTo(second.name)); |
| print(''); |
| printHeading(2, 'The worst completion results'); |
| for (var group in groups) { |
| _printWorstResults('In ${group.name}', worstResults[group]); |
| } |
| } |
| |
| int _computeCharsBeforeTop(ExpectedCompletion target, |
| List<protocol.CompletionSuggestion> suggestions, |
| {int minRank = 1}) { |
| var rank = placementInSuggestionList(suggestions, target).rank; |
| if (rank <= minRank) { |
| return 0; |
| } |
| var expected = target.completion; |
| for (var i = 1; i < expected.length + 1; i++) { |
| var prefix = expected.substring(0, i); |
| var filteredSuggestions = _filterSuggestions(prefix, suggestions); |
| rank = placementInSuggestionList(filteredSuggestions, target).rank; |
| if (rank <= minRank) { |
| return i; |
| } |
| } |
| return expected.length; |
| } |
| |
| Future<List<protocol.CompletionSuggestion>> _computeCompletionSuggestions( |
| MetricsSuggestionListener listener, |
| OperationPerformanceImpl performance, |
| CompletionRequestImpl request, |
| [DeclarationsTracker declarationsTracker, |
| protocol.CompletionAvailableSuggestionsParams |
| availableSuggestionsParams]) async { |
| List<protocol.CompletionSuggestion> suggestions; |
| |
| if (declarationsTracker == null) { |
| // available suggestions == false |
| suggestions = await DartCompletionManager( |
| dartdocDirectiveInfo: DartdocDirectiveInfo(), |
| listener: listener, |
| ).computeSuggestions(performance, request); |
| } else { |
| // available suggestions == true |
| var includedElementKinds = <protocol.ElementKind>{}; |
| var includedElementNames = <String>{}; |
| var includedSuggestionRelevanceTagList = |
| <protocol.IncludedSuggestionRelevanceTag>[]; |
| var includedSuggestionSetList = <protocol.IncludedSuggestionSet>[]; |
| suggestions = await DartCompletionManager( |
| dartdocDirectiveInfo: DartdocDirectiveInfo(), |
| includedElementKinds: includedElementKinds, |
| includedElementNames: includedElementNames, |
| includedSuggestionRelevanceTags: includedSuggestionRelevanceTagList, |
| listener: listener, |
| ).computeSuggestions(performance, request); |
| |
| computeIncludedSetList(declarationsTracker, request.result, |
| includedSuggestionSetList, includedElementNames); |
| |
| var includedSuggestionSetMap = { |
| for (var includedSuggestionSet in includedSuggestionSetList) |
| includedSuggestionSet.id: includedSuggestionSet, |
| }; |
| |
| var includedSuggestionRelevanceTagMap = { |
| for (var includedSuggestionRelevanceTag |
| in includedSuggestionRelevanceTagList) |
| includedSuggestionRelevanceTag.tag: |
| includedSuggestionRelevanceTag.relevanceBoost, |
| }; |
| |
| for (var availableSuggestionSet |
| in availableSuggestionsParams.changedLibraries) { |
| var id = availableSuggestionSet.id; |
| for (var availableSuggestion in availableSuggestionSet.items) { |
| // Exclude available suggestions where this element kind doesn't match |
| // an element kind in includedElementKinds. |
| var elementKind = availableSuggestion.element?.kind; |
| if (elementKind != null && |
| includedElementKinds.contains(elementKind)) { |
| if (includedSuggestionSetMap.containsKey(id)) { |
| var relevance = includedSuggestionSetMap[id].relevance; |
| |
| // Search for any matching relevance tags to apply any boosts |
| if (includedSuggestionRelevanceTagList.isNotEmpty && |
| availableSuggestion.relevanceTags != null && |
| availableSuggestion.relevanceTags.isNotEmpty) { |
| for (var tag in availableSuggestion.relevanceTags) { |
| if (includedSuggestionRelevanceTagMap.containsKey(tag)) { |
| // apply the boost |
| relevance += includedSuggestionRelevanceTagMap[tag]; |
| } |
| } |
| } |
| suggestions |
| .add(availableSuggestion.toCompletionSuggestion(relevance)); |
| } |
| } |
| } |
| } |
| } |
| |
| suggestions.sort(completionComparator); |
| return suggestions; |
| } |
| |
| /// Compute the metrics for the files in the context [root], creating a |
| /// separate context collection to prevent accumulating memory. The metrics |
| /// should be captured in the [collector]. |
| Future<void> _computeInContext(ContextRoot root) async { |
| // Create a new collection to avoid consuming large quantities of memory. |
| final collection = AnalysisContextCollectionImpl( |
| includedPaths: root.includedPaths.toList(), |
| excludedPaths: root.excludedPaths.toList(), |
| resourceProvider: _provider, |
| ); |
| |
| var context = collection.contexts[0]; |
| |
| // Set the DeclarationsTracker, only call doWork to build up the available |
| // suggestions if doComputeCompletionsFromAnalysisServer is true. |
| DeclarationsTracker declarationsTracker; |
| protocol.CompletionAvailableSuggestionsParams availableSuggestionsParams; |
| if (options.availableSuggestions) { |
| declarationsTracker = DeclarationsTracker( |
| MemoryByteStore(), PhysicalResourceProvider.INSTANCE); |
| declarationsTracker.addContext(context); |
| while (declarationsTracker.hasWork) { |
| declarationsTracker.doWork(); |
| } |
| |
| // Have the AvailableDeclarationsSet computed to use later. |
| availableSuggestionsParams = createCompletionAvailableSuggestions( |
| declarationsTracker.allLibraries.toList(), []); |
| |
| // assert that this object is not null, throw if it is. |
| if (availableSuggestionsParams == null) { |
| throw Exception('availableSuggestionsParam not computable.'); |
| } |
| } |
| |
| // Loop through each file, resolve the file and call |
| // forEachExpectedCompletion |
| var pathContext = context.contextRoot.resourceProvider.pathContext; |
| for (var filePath in context.contextRoot.analyzedFiles()) { |
| if (file_paths.isDart(pathContext, filePath)) { |
| try { |
| _resolvedUnitResult = |
| await context.currentSession.getResolvedUnit(filePath); |
| |
| var analysisError = getFirstErrorOrNull(_resolvedUnitResult); |
| if (analysisError != null) { |
| print('File $filePath skipped due to errors such as:'); |
| print(' ${analysisError.toString()}'); |
| print(''); |
| resultCode = 1; |
| continue; |
| } |
| |
| // Use the ExpectedCompletionsVisitor to compute the set of expected |
| // completions for this CompilationUnit. |
| final visitor = ExpectedCompletionsVisitor(filePath); |
| _resolvedUnitResult.unit.accept(visitor); |
| |
| for (var expectedCompletion in visitor.expectedCompletions) { |
| var resolvedUnitResult = _resolvedUnitResult; |
| |
| // If an overlay option is being used, compute the overlay file, and |
| // have the context reanalyze the file |
| if (options.overlay != CompletionMetricsOptions.OVERLAY_NONE) { |
| var overlayContents = _getOverlayContents( |
| _resolvedUnitResult.content, expectedCompletion); |
| |
| _provider.setOverlay(filePath, |
| content: overlayContents, |
| modificationStamp: overlayModificationStamp++); |
| context.driver.changeFile(filePath); |
| resolvedUnitResult = |
| await context.currentSession.getResolvedUnit(filePath); |
| } |
| |
| // As this point the completion suggestions are computed, |
| // and results are collected with varying settings for |
| // comparison: |
| |
| Future<int> handleExpectedCompletion( |
| {MetricsSuggestionListener listener, |
| @required CompletionMetrics metrics}) async { |
| var stopwatch = Stopwatch()..start(); |
| var request = CompletionRequestImpl( |
| resolvedUnitResult, |
| expectedCompletion.offset, |
| CompletionPerformance(), |
| ); |
| var directiveInfo = DartdocDirectiveInfo(); |
| |
| OpType opType; |
| List<protocol.CompletionSuggestion> suggestions; |
| await request.performance.runRequestOperation( |
| (performance) async { |
| var dartRequest = await DartCompletionRequestImpl.from( |
| performance, request, directiveInfo); |
| opType = |
| OpType.forCompletion(dartRequest.target, request.offset); |
| suggestions = await _computeCompletionSuggestions( |
| listener, |
| performance, |
| request, |
| declarationsTracker, |
| availableSuggestionsParams, |
| ); |
| }, |
| ); |
| stopwatch.stop(); |
| |
| return forEachExpectedCompletion( |
| request, |
| listener, |
| expectedCompletion, |
| opType.completionLocation, |
| suggestions, |
| metrics, |
| stopwatch.elapsedMilliseconds); |
| } |
| |
| var bestRank = -1; |
| var bestName = ''; |
| for (var metrics in targetMetrics) { |
| // Compute the completions. |
| metrics.enable(); |
| // if (FeatureComputer.noDisabledFeatures) { |
| // var line = expectedCompletion.lineNumber; |
| // var column = expectedCompletion.columnNumber; |
| // print('$filePath:$line:$column'); |
| // } |
| var listener = MetricsSuggestionListener(); |
| var rank = await handleExpectedCompletion( |
| listener: listener, metrics: metrics); |
| if (bestRank < 0 || rank < bestRank) { |
| bestRank = rank; |
| bestName = metrics.name; |
| } |
| metrics.disable(); |
| } |
| rankComparison.count(bestName); |
| |
| // If an overlay option is being used, remove the overlay applied |
| // earlier |
| if (options.overlay != CompletionMetricsOptions.OVERLAY_NONE) { |
| _provider.removeOverlay(filePath); |
| } |
| } |
| } catch (exception, stackTrace) { |
| print('Exception caught analyzing: $filePath'); |
| print(exception.toString()); |
| print(stackTrace); |
| resultCode = 1; |
| } |
| } |
| } |
| } |
| |
| List<protocol.CompletionSuggestion> _filterSuggestions( |
| String prefix, List<protocol.CompletionSuggestion> suggestions) { |
| // TODO(brianwilkerson) Replace this with a more realistic filtering |
| // algorithm. |
| return suggestions |
| .where((suggestion) => suggestion.completion.startsWith(prefix)) |
| .toList(); |
| } |
| |
| String _getOverlayContents( |
| String contents, ExpectedCompletion expectedCompletion) { |
| assert(contents.isNotEmpty); |
| var offset = expectedCompletion.offset; |
| var length = expectedCompletion.syntacticEntity.length; |
| assert(offset >= 0); |
| assert(length > 0); |
| if (options.overlay == CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN) { |
| return contents.substring(0, offset) + |
| contents.substring(offset + length); |
| } else if (options.overlay == |
| CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE) { |
| return contents.substring(0, offset); |
| } else { |
| var removeToken = CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN; |
| var removeRest = CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE; |
| throw Exception('\'_getOverlayContents\' called with option other than' |
| '$removeToken and $removeRest: ${options.overlay}'); |
| } |
| } |
| |
| void _printSlowestResults( |
| String title, List<CompletionResult> slowestResults) { |
| printHeading(3, title); |
| var needsBlankLine = false; |
| for (var result in slowestResults) { |
| var elapsedMS = result.elapsedMS; |
| var expected = result.expectedCompletion; |
| if (needsBlankLine) { |
| print(''); |
| } else { |
| needsBlankLine = true; |
| } |
| print(' Elapsed ms: $elapsedMS'); |
| print(' Completion: ${expected.completion}'); |
| print(' Completion kind: ${expected.kind}'); |
| print(' Element kind: ${expected.elementKind}'); |
| print(' Location: ${expected.location}'); |
| } |
| } |
| |
| void _printWorstResults(String title, List<CompletionResult> worstResults) { |
| List<String> suggestionRow(int rank, SuggestionData data) { |
| var suggestion = data.suggestion; |
| return [ |
| rank.toString(), |
| suggestion.relevance.toString(), |
| suggestion.completion, |
| suggestion.kind.toString() |
| ]; |
| } |
| |
| List<String> featuresRow(int rank, SuggestionData data) { |
| var features = data.features; |
| return [ |
| rank.toString(), |
| for (var feature in features) feature.toStringAsFixed(4) |
| ]; |
| } |
| |
| printHeading(3, title); |
| var needsBlankLine = false; |
| for (var result in worstResults) { |
| var rank = result.place.rank; |
| var actualSuggestion = result.actualSuggestion; |
| var expected = result.expectedCompletion; |
| |
| var topSuggestions = result.topSuggestions; |
| var topSuggestionCount = topSuggestions.length; |
| |
| var preceding = result.precedingRelevanceCounts; |
| var precedingRelevances = preceding.keys.toList(); |
| precedingRelevances.sort(); |
| |
| var suggestionsTable = [ |
| ['Rank', 'Relevance', 'Completion', 'Kind'] |
| ]; |
| for (var i = 0; i < topSuggestionCount; i++) { |
| suggestionsTable.add(suggestionRow(i, topSuggestions[i])); |
| } |
| suggestionsTable.add(suggestionRow(rank, actualSuggestion)); |
| rightJustifyColumns(suggestionsTable, [0, 1]); |
| |
| var featuresTable = [ |
| [ |
| 'Rank', |
| 'contextType', |
| 'elementKind', |
| 'hasDeprecated', |
| 'inheritanceDistance', |
| 'isConstant', |
| 'isNoSuchMethod', |
| 'keyword', |
| 'localVariableDistance', |
| 'startsWithDollar', |
| 'superMatches' |
| ] |
| ]; |
| for (var i = 0; i < topSuggestionCount; i++) { |
| featuresTable.add(featuresRow(i, topSuggestions[i])); |
| } |
| featuresTable.add(featuresRow(rank, actualSuggestion)); |
| rightJustifyColumns(featuresTable, range(0, featuresTable[0].length)); |
| |
| if (needsBlankLine) { |
| print(''); |
| } else { |
| needsBlankLine = true; |
| } |
| print(' Rank: $rank'); |
| print(' Location: ${expected.location}'); |
| print(' Comparison with the top $topSuggestionCount suggestions:'); |
| printTable(suggestionsTable); |
| print(' Comparison of features with the top $topSuggestionCount ' |
| 'suggestions:'); |
| printTable(featuresTable); |
| print(' Preceding relevance scores and counts:'); |
| for (var relevance in precedingRelevances.reversed) { |
| print(' $relevance: ${preceding[relevance]}'); |
| } |
| } |
| } |
| |
| /// Given some [ResolvedUnitResult] return the first error of high severity |
| /// if such an error exists, `null` otherwise. |
| static err.AnalysisError getFirstErrorOrNull( |
| ResolvedUnitResult resolvedUnitResult) { |
| for (var error in resolvedUnitResult.errors) { |
| if (error.severity == Severity.error) { |
| return error; |
| } |
| } |
| return null; |
| } |
| |
| static Place placementInSuggestionList( |
| List<protocol.CompletionSuggestion> suggestions, |
| ExpectedCompletion expectedCompletion) { |
| var placeCounter = 1; |
| for (var completionSuggestion in suggestions) { |
| if (expectedCompletion.matches(completionSuggestion)) { |
| return Place(placeCounter, suggestions.length); |
| } |
| placeCounter++; |
| } |
| return Place.none(); |
| } |
| } |
| |
| /// The options specified on the command-line. |
| class CompletionMetricsOptions { |
| /// A flag that causes the available suggestion sets to be used while |
| /// computing suggestions. |
| static const String AVAILABLE_SUGGESTIONS = 'available-suggestions'; |
| |
| /// An option to control whether and how overlays should be produced. |
| static const String OVERLAY = 'overlay'; |
| |
| /// A mode indicating that no overlays should be produced. |
| static const String OVERLAY_NONE = 'none'; |
| |
| /// A mode indicating that everything from the completion offset to the end of |
| /// the file should be removed. |
| static const String OVERLAY_REMOVE_REST_OF_FILE = 'remove-rest-of-file'; |
| |
| /// A mode indicating that the token whose offset is the same as the |
| /// completion offset should be removed. |
| static const String OVERLAY_REMOVE_TOKEN = 'remove-token'; |
| |
| /// A flag that causes detailed information to be printed every time a |
| /// completion request fails to produce a suggestions matching the expected |
| /// suggestion. |
| static const String PRINT_MISSED_COMPLETION_DETAILS = |
| 'print-missed-completion-details'; |
| |
| /// A flag that causes summary information to be printed about the times that |
| /// a completion request failed to produce a suggestions matching the expected |
| /// suggestion. |
| static const String PRINT_MISSED_COMPLETION_SUMMARY = |
| 'print-missed-completion-summary'; |
| |
| /// A flag that causes information to be printed about places where no |
| /// completion location was computed and about information that's missing in |
| /// the completion tables. |
| static const String PRINT_MISSING_INFORMATION = 'print-missing-information'; |
| |
| /// A flag that causes information to be printed about the mrr score achieved |
| /// at each completion location. |
| static const String PRINT_MRR_BY_LOCATION = 'print-mrr-by-location'; |
| |
| /// A flag that causes information to be printed about the completion requests |
| /// that were the slowest to return suggestions. |
| static const String PRINT_SLOWEST_RESULTS = 'print-slowest-results'; |
| |
| /// A flag that causes information to be printed about the completion requests |
| /// that had the worst mrr scores. |
| static const String PRINT_WORST_RESULTS = 'print-worst-results'; |
| |
| /// A flag indicating whether available suggestions should be enabled for this |
| /// run. |
| final bool availableSuggestions; |
| |
| /// The overlay mode that should be used. |
| final String overlay; |
| |
| /// A flag indicating whether information should be printed every time a |
| /// completion request fails to produce a suggestions matching the expected |
| /// suggestion. |
| final bool printMissedCompletionDetails; |
| |
| /// A flag indicating whether information should be printed every time a |
| /// completion request fails to produce a suggestions matching the expected |
| /// suggestion. |
| final bool printMissedCompletionSummary; |
| |
| /// A flag indicating whether information should be printed about places where |
| /// no completion location was computed and about information that's missing |
| /// in the completion tables. |
| final bool printMissingInformation; |
| |
| /// A flag indicating whether information should be printed about the mrr |
| /// score achieved at each completion location. |
| final bool printMrrByLocation; |
| |
| /// A flag indicating whether information should be printed about the |
| /// completion requests that were the slowest to return suggestions. |
| final bool printSlowestResults; |
| |
| /// A flag indicating whether information should be printed about the |
| /// completion requests that had the worst mrr scores. |
| final bool printWorstResults; |
| |
| factory CompletionMetricsOptions(results) { |
| return CompletionMetricsOptions._( |
| availableSuggestions: results[AVAILABLE_SUGGESTIONS], |
| overlay: results[OVERLAY], |
| printMissedCompletionDetails: results[PRINT_MISSED_COMPLETION_DETAILS], |
| printMissedCompletionSummary: results[PRINT_MISSED_COMPLETION_SUMMARY], |
| printMissingInformation: results[PRINT_MISSING_INFORMATION], |
| printMrrByLocation: results[PRINT_MRR_BY_LOCATION], |
| printSlowestResults: results[PRINT_SLOWEST_RESULTS], |
| printWorstResults: results[PRINT_WORST_RESULTS]); |
| } |
| |
| CompletionMetricsOptions._( |
| {@required this.availableSuggestions, |
| @required this.overlay, |
| @required this.printMissedCompletionDetails, |
| @required this.printMissedCompletionSummary, |
| @required this.printMissingInformation, |
| @required this.printMrrByLocation, |
| @required this.printSlowestResults, |
| @required this.printWorstResults}) |
| : assert(overlay == OVERLAY_NONE || |
| overlay == OVERLAY_REMOVE_TOKEN || |
| overlay == OVERLAY_REMOVE_REST_OF_FILE); |
| } |
| |
| /// The result of a single completion. |
| class CompletionResult { |
| final Place place; |
| |
| final CompletionRequestImpl request; |
| |
| final SuggestionData actualSuggestion; |
| |
| final List<SuggestionData> topSuggestions; |
| |
| final ExpectedCompletion expectedCompletion; |
| |
| final String completionLocation; |
| |
| final int elapsedMS; |
| |
| final Map<int, int> precedingRelevanceCounts; |
| |
| CompletionResult( |
| this.place, |
| this.request, |
| this.actualSuggestion, |
| this.topSuggestions, |
| this.precedingRelevanceCounts, |
| this.expectedCompletion, |
| this.completionLocation, |
| this.elapsedMS); |
| |
| /// Return an instance extracted from the decoded JSON [map]. |
| factory CompletionResult.fromJson(Map<String, dynamic> map) { |
| var place = Place.fromJson(map['place'] as Map<String, dynamic>); |
| var actualSuggestion = SuggestionData.fromJson( |
| map['actualSuggestion'] as Map<String, dynamic>); |
| var topSuggestions = (map['topSuggestions'] as List<dynamic>) |
| ?.map((map) => SuggestionData.fromJson(map as Map<String, dynamic>)) |
| ?.toList(); |
| var precedingRelevanceCounts = |
| (map['precedingRelevanceCounts'] as Map<String, dynamic>) |
| ?.map((key, value) => MapEntry(int.parse(key), value as int)); |
| var expectedCompletion = ExpectedCompletion.fromJson( |
| map['expectedCompletion'] as Map<String, dynamic>); |
| var completionLocation = map['completionLocation'] as String; |
| var elapsedMS = map['elapsedMS'] as int; |
| return CompletionResult( |
| place, |
| null, |
| actualSuggestion, |
| topSuggestions, |
| precedingRelevanceCounts, |
| expectedCompletion, |
| completionLocation, |
| elapsedMS); |
| } |
| |
| /// Return the completion group for the location at which completion was |
| /// requested. |
| CompletionGroup get group { |
| var entity = expectedCompletion.syntacticEntity; |
| var element = _getElement(entity); |
| if (element != null) { |
| var parent = element.enclosingElement; |
| if (parent is ClassElement || parent is ExtensionElement) { |
| if (_isStatic(element)) { |
| return CompletionGroup.staticMember; |
| } else { |
| return CompletionGroup.instanceMember; |
| } |
| } else if (parent is CompilationUnitElement && |
| element is! ClassElement && |
| element is! ExtensionElement) { |
| return CompletionGroup.topLevelMember; |
| } |
| if (element is ClassElement) { |
| if (element.isEnum) { |
| return CompletionGroup.enumElement; |
| } else if (element.isMixin) { |
| return CompletionGroup.mixinElement; |
| } |
| if (entity is SimpleIdentifier && |
| entity.parent is TypeName && |
| entity.parent.parent is ConstructorName && |
| entity.parent.parent.parent is InstanceCreationExpression) { |
| return CompletionGroup.constructorElement; |
| } |
| return CompletionGroup.classElement; |
| } else if (element is ExtensionElement) { |
| return CompletionGroup.extensionElement; |
| } else if (element is FunctionElement) { |
| return CompletionGroup.localFunctionElement; |
| } else if (element is LocalVariableElement) { |
| return CompletionGroup.localVariableElement; |
| } else if (element is ParameterElement) { |
| return CompletionGroup.parameterElement; |
| } else if (element is PrefixElement) { |
| return CompletionGroup.prefixElement; |
| } else if (element is TypeParameterElement) { |
| return CompletionGroup.typeParameterElement; |
| } |
| } |
| if (entity is SimpleIdentifier) { |
| var name = entity.name; |
| if (name == 'void') { |
| return CompletionGroup.keywordVoid; |
| } else if (name == 'dynamic') { |
| return CompletionGroup.keywordDynamic; |
| } |
| } |
| return CompletionGroup.unknown; |
| } |
| |
| /// Return a map used to represent this completion result in a JSON structure. |
| Map<String, dynamic> toJson() { |
| return { |
| 'place': place.toJson(), |
| 'actualSuggestion': actualSuggestion.toJson(), |
| if (topSuggestions != null) |
| 'topSuggestions': |
| topSuggestions.map((suggestion) => suggestion.toJson()).toList(), |
| if (precedingRelevanceCounts != null) |
| 'precedingRelevanceCounts': precedingRelevanceCounts |
| .map((key, value) => MapEntry(key.toString(), value)), |
| 'expectedCompletion': expectedCompletion.toJson(), |
| 'completionLocation': completionLocation, |
| 'elapsedMS': elapsedMS, |
| }; |
| } |
| |
| /// Return the element associated with the syntactic [entity], or `null` if |
| /// there is no such element. |
| Element _getElement(SyntacticEntity entity) { |
| if (entity is SimpleIdentifier) { |
| var element = entity.staticElement; |
| if (element != null) { |
| return element; |
| } |
| AstNode node = entity; |
| while (node != null) { |
| var parent = node.parent; |
| if (parent is AssignmentExpression) { |
| if (node == parent.leftHandSide) { |
| return parent.readElement ?? parent.writeElement; |
| } |
| return null; |
| } else if (parent is PrefixExpression) { |
| if (parent.operator.type == TokenType.PLUS_PLUS || |
| parent.operator.type == TokenType.MINUS_MINUS) { |
| return parent.readElement ?? parent.writeElement; |
| } |
| } else if (parent is PostfixExpression) { |
| return parent.readElement ?? parent.writeElement; |
| } |
| node = parent; |
| } |
| } |
| return null; |
| } |
| |
| /// Return `true` if the [element] is static (either top-level or a static |
| /// member of a class or extension). |
| bool _isStatic(Element element) { |
| if (element is ClassMemberElement) { |
| return element.isStatic; |
| } else if (element is ExecutableElement) { |
| return element.isStatic; |
| } else if (element is FieldElement) { |
| return element.isStatic; |
| } else if (element is VariableElement) { |
| return element.isStatic; |
| } |
| return true; |
| } |
| } |
| |
| /// The data to be printed on a single line in the table of mrr values per |
| /// completion location. |
| class LocationTableLine { |
| final String label; |
| final double product; |
| final int count; |
| final double mrr; |
| final double mrr_5; |
| |
| LocationTableLine( |
| {@required this.label, |
| @required this.product, |
| @required this.count, |
| @required this.mrr, |
| @required this.mrr_5}); |
| } |
| |
| class MetricsSuggestionListener implements SuggestionListener { |
| Map<protocol.CompletionSuggestion, List<double>> featureMap = {}; |
| |
| List<double> cachedFeatures = const [ |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0, |
| 0.0 |
| ]; |
| |
| String missingCompletionLocation; |
| |
| String missingCompletionLocationTable; |
| |
| @override |
| void builtSuggestion(protocol.CompletionSuggestion suggestion) { |
| featureMap[suggestion] = cachedFeatures; |
| cachedFeatures = const [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]; |
| } |
| |
| @override |
| void computedFeatures( |
| {double contextType = 0.0, |
| double elementKind = 0.0, |
| double hasDeprecated = 0.0, |
| double inheritanceDistance = 0.0, |
| double isConstant = 0.0, |
| double isNoSuchMethod = 0.0, |
| double keyword = 0.0, |
| double localVariableDistance = 0.0, |
| double startsWithDollar = 0.0, |
| double superMatches = 0.0}) { |
| cachedFeatures = [ |
| contextType, |
| elementKind, |
| hasDeprecated, |
| inheritanceDistance, |
| isConstant, |
| isNoSuchMethod, |
| keyword, |
| localVariableDistance, |
| startsWithDollar, |
| superMatches |
| ]; |
| } |
| |
| @override |
| void missingCompletionLocationAt(AstNode parent, SyntacticEntity child) { |
| if (missingCompletionLocation == null) { |
| String className(SyntacticEntity entity) { |
| var className = entity.runtimeType.toString(); |
| if (className.endsWith('Impl')) { |
| className = className.substring(0, className.length - 4); |
| } |
| return className; |
| } |
| |
| var parentClass = className(parent); |
| var childClass = className(child); |
| missingCompletionLocation = '$parentClass/$childClass'; |
| } |
| } |
| |
| @override |
| void missingElementKindTableFor(String completionLocation) { |
| missingCompletionLocationTable = completionLocation; |
| } |
| } |
| |
| /// A description of a pair of relevance tables to be used in an experiment. |
| class RelevanceTables { |
| /// The name of the experiment using the tables. |
| final String name; |
| |
| /// The relevance table used for element kinds. |
| final Map<String, Map<protocol.ElementKind, ProbabilityRange>> |
| elementKindRelevance; |
| |
| /// The relevance table used for keywords. |
| final Map<String, Map<String, ProbabilityRange>> keywordRelevance; |
| |
| /// Initialize a newly created description of a pair of relevance tables. |
| RelevanceTables(this.name, this.elementKindRelevance, this.keywordRelevance); |
| } |
| |
| /// The information being remembered about an individual suggestion. |
| class SuggestionData { |
| /// The suggestion that was produced. |
| protocol.CompletionSuggestion suggestion; |
| |
| /// The values of the features used to compute the suggestion. |
| List<double> features; |
| |
| SuggestionData(this.suggestion, this.features); |
| |
| /// Return an instance extracted from the decoded JSON [map]. |
| factory SuggestionData.fromJson(Map<String, dynamic> map) { |
| return SuggestionData( |
| protocol.CompletionSuggestion.fromJson(ResponseDecoder(null), '', |
| map['suggestion'] as Map<String, dynamic>), |
| (map['features'] as List<dynamic>).cast<double>()); |
| } |
| |
| /// Return a map used to represent this suggestion data in a JSON structure. |
| Map<String, dynamic> toJson() { |
| return { |
| 'suggestion': suggestion.toJson(), |
| 'features': features, |
| }; |
| } |
| } |
| |
| extension on CompletionGroup { |
| String get name { |
| switch (this) { |
| case CompletionGroup.classElement: |
| return 'class'; |
| case CompletionGroup.constructorElement: |
| return 'constructor'; |
| case CompletionGroup.enumElement: |
| return 'enum'; |
| case CompletionGroup.extensionElement: |
| return 'extension'; |
| case CompletionGroup.instanceMember: |
| return 'instance member'; |
| case CompletionGroup.labelElement: |
| return 'label'; |
| case CompletionGroup.localFunctionElement: |
| return 'local function'; |
| case CompletionGroup.localVariableElement: |
| return 'local variable'; |
| case CompletionGroup.mixinElement: |
| return 'mixin'; |
| case CompletionGroup.parameterElement: |
| return 'parameter'; |
| case CompletionGroup.prefixElement: |
| return 'prefix'; |
| case CompletionGroup.staticMember: |
| return 'static member'; |
| case CompletionGroup.topLevelMember: |
| return 'top level member'; |
| case CompletionGroup.typeParameterElement: |
| return 'type parameter'; |
| |
| case CompletionGroup.keywordDynamic: |
| return 'keyword dynamic'; |
| case CompletionGroup.keywordVoid: |
| return 'keyword void'; |
| |
| case CompletionGroup.unknown: |
| return 'unknown'; |
| } |
| return '<unknown>'; |
| } |
| } |
| |
| extension AvailableSuggestionsExtension on protocol.AvailableSuggestion { |
| // TODO(jwren) I am not sure if we want CompletionSuggestionKind.INVOCATION in |
| // call cases here, to iterate I need to figure out why this algorithm is |
| // taking so much time. |
| protocol.CompletionSuggestion toCompletionSuggestion(int relevance) => |
| protocol.CompletionSuggestion( |
| protocol.CompletionSuggestionKind.INVOCATION, |
| relevance, |
| label, |
| label.length, |
| 0, |
| element.isDeprecated, |
| false); |
| } |
