Minor improvements to the code completion metrics tool
It now captures and prints the minimum and maximum values for several
characteristics. It also now allows the flag controlling whether the
available suggestion sets are used to be set for each experiment rather
than globally (which allows us to now compare the results with and
without the flag set).
Change-Id: I5b5930216ff2f448de35f6dbf3ccd9de712179d1
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/196564
Commit-Queue: Brian Wilkerson <brianwilkerson@google.com>
Reviewed-by: Konstantin Shcheglov <scheglov@google.com>
diff --git a/pkg/analysis_server/tool/code_completion/completion_metrics.dart b/pkg/analysis_server/tool/code_completion/completion_metrics.dart
index 4a685e3..596d170 100644
--- a/pkg/analysis_server/tool/code_completion/completion_metrics.dart
+++ b/pkg/analysis_server/tool/code_completion/completion_metrics.dart
@@ -123,16 +123,6 @@
/// Create a parser that can be used to parse the command-line arguments.
ArgParser createArgParser() {
return ArgParser()
- ..addFlag(CompletionMetricsOptions.AVAILABLE_SUGGESTIONS,
- abbr: 'a',
- help:
- 'Use the available suggestions feature in the Analysis Server when '
- 'computing the set of code completions. With this feature enabled, '
- 'completion will match the support in the Dart Plugin for '
- 'IntelliJ, without this enabled the completion support matches the '
- 'support in LSP.',
- defaultsTo: false,
- negatable: false)
..addOption(
'help',
abbr: 'h',
@@ -282,6 +272,10 @@
/// The name associated with this set of metrics.
final String name;
+ /// A flag indicating whether available suggestions should be enabled for this
+ /// run.
+ final bool availableSuggestions;
+
/// The function to be executed when this metrics collector is enabled.
final void Function()? enableFunction;
@@ -336,11 +330,15 @@
final Map<CompletionGroup, List<CompletionResult>> worstResults = {};
- CompletionMetrics(this.name, {this.enableFunction, this.disableFunction});
+ CompletionMetrics(this.name,
+ {required this.availableSuggestions,
+ this.enableFunction,
+ this.disableFunction});
/// Return an instance extracted from the decoded JSON [map].
factory CompletionMetrics.fromJson(Map<String, dynamic> map) {
- var metrics = CompletionMetrics(map['name'] as String);
+ var metrics = CompletionMetrics(map['name'] as String,
+ availableSuggestions: map['availableSuggestions'] as bool);
metrics.completionCounter
.fromJson(map['completionCounter'] as Map<String, dynamic>);
metrics.completionMissedTokenCounter
@@ -467,6 +465,7 @@
Map<String, dynamic> toJson() {
return {
'name': name,
+ 'availableSuggestions': availableSuggestions,
'completionCounter': completionCounter.toJson(),
'completionMissedTokenCounter': completionMissedTokenCounter.toJson(),
'completionKindCounter': completionKindCounter.toJson(),
@@ -583,31 +582,43 @@
CompletionMetricsComputer(this.rootPath, this.options);
/// Compare the metrics when each feature is used in isolation.
- void compareIndividualFeatures() {
+ void compareIndividualFeatures({bool availableSuggestions = false}) {
var featureNames = FeatureComputer.featureNames;
var featureCount = featureNames.length;
for (var i = 0; i < featureCount; i++) {
var weights = List.filled(featureCount, 0.00);
weights[i] = 1.00;
- targetMetrics.add(CompletionMetrics(featureNames[i], enableFunction: () {
- FeatureComputer.featureWeights = weights;
- }, disableFunction: () {
- FeatureComputer.featureWeights = FeatureComputer.defaultFeatureWeights;
- }));
+ targetMetrics.add(CompletionMetrics(
+ featureNames[i],
+ availableSuggestions: availableSuggestions,
+ enableFunction: () {
+ FeatureComputer.featureWeights = weights;
+ },
+ disableFunction: () {
+ FeatureComputer.featureWeights =
+ FeatureComputer.defaultFeatureWeights;
+ },
+ ));
}
}
/// Compare the relevance [tables] to the default relevance tables.
- void compareRelevanceTables(List<RelevanceTables> tables) {
+ void compareRelevanceTables(List<RelevanceTables> tables,
+ {bool availableSuggestions = false}) {
assert(tables.isNotEmpty);
for (var tablePair in tables) {
- targetMetrics.add(CompletionMetrics(tablePair.name, enableFunction: () {
- elementKindRelevance = tablePair.elementKindRelevance;
- keywordRelevance = tablePair.keywordRelevance;
- }, disableFunction: () {
- elementKindRelevance = defaultElementKindRelevance;
- keywordRelevance = defaultKeywordRelevance;
- }));
+ targetMetrics.add(CompletionMetrics(
+ tablePair.name,
+ availableSuggestions: availableSuggestions,
+ enableFunction: () {
+ elementKindRelevance = tablePair.elementKindRelevance;
+ keywordRelevance = tablePair.keywordRelevance;
+ },
+ disableFunction: () {
+ elementKindRelevance = defaultElementKindRelevance;
+ keywordRelevance = defaultKeywordRelevance;
+ },
+ ));
}
}
@@ -616,16 +627,18 @@
// To compare two or more changes to completions, add a `CompletionMetrics`
// object with enable and disable functions to the list of `targetMetrics`.
targetMetrics.add(CompletionMetrics('shipping',
- enableFunction: null, disableFunction: null));
+ availableSuggestions: false,
+ enableFunction: null,
+ disableFunction: null));
// To compare two or more relevance tables, uncomment the line below and
// add the `RelevanceTables` to the list. The default relevance tables
// should not be included in the list.
-// compareRelevanceTables([]);
+// compareRelevanceTables([], availableSuggestions: false);
// To compare the relative benefit from each of the features, uncomment the
// line below.
-// compareIndividualFeatures();
+// compareIndividualFeatures(availableSuggestions: false);
final collection = AnalysisContextCollectionImpl(
includedPaths: [rootPath],
@@ -916,7 +929,10 @@
var computers = sources.toList();
var row = [computers.first.name];
for (var computer in computers) {
- row.add(computer.mean.toStringAsFixed(6));
+ var min = computer.min;
+ var mean = computer.mean.toStringAsFixed(6);
+ var max = computer.max;
+ row.add('$min, $mean, $max');
}
return row;
}
@@ -1097,7 +1113,7 @@
// suggestions if doComputeCompletionsFromAnalysisServer is true.
DeclarationsTracker? declarationsTracker;
protocol.CompletionAvailableSuggestionsParams? availableSuggestionsParams;
- if (options.availableSuggestions) {
+ if (targetMetrics.any((metrics) => metrics.availableSuggestions)) {
declarationsTracker = DeclarationsTracker(
MemoryByteStore(), PhysicalResourceProvider.INSTANCE);
declarationsTracker.addContext(context);
@@ -1177,8 +1193,10 @@
listener,
performance,
request,
- declarationsTracker,
- availableSuggestionsParams,
+ metrics.availableSuggestions ? declarationsTracker : null,
+ metrics.availableSuggestions
+ ? availableSuggestionsParams
+ : null,
);
},
);
@@ -1391,10 +1409,6 @@
/// The options specified on the command-line.
class CompletionMetricsOptions {
- /// A flag that causes the available suggestion sets to be used while
- /// computing suggestions.
- static const String AVAILABLE_SUGGESTIONS = 'available-suggestions';
-
/// An option to control whether and how overlays should be produced.
static const String OVERLAY = 'overlay';
@@ -1438,10 +1452,6 @@
/// that had the worst mrr scores.
static const String PRINT_WORST_RESULTS = 'print-worst-results';
- /// A flag indicating whether available suggestions should be enabled for this
- /// run.
- final bool availableSuggestions;
-
/// The overlay mode that should be used.
final String overlay;
@@ -1474,7 +1484,6 @@
factory CompletionMetricsOptions(results) {
return CompletionMetricsOptions._(
- availableSuggestions: results[AVAILABLE_SUGGESTIONS],
overlay: results[OVERLAY],
printMissedCompletionDetails: results[PRINT_MISSED_COMPLETION_DETAILS],
printMissedCompletionSummary: results[PRINT_MISSED_COMPLETION_SUMMARY],
@@ -1485,8 +1494,7 @@
}
CompletionMetricsOptions._(
- {required this.availableSuggestions,
- required this.overlay,
+ {required this.overlay,
required this.printMissedCompletionDetails,
required this.printMissedCompletionSummary,
required this.printMissingInformation,
diff --git a/pkg/analysis_server/tool/code_completion/metrics_util.dart b/pkg/analysis_server/tool/code_completion/metrics_util.dart
index 00364e9..c3b01717 100644
--- a/pkg/analysis_server/tool/code_completion/metrics_util.dart
+++ b/pkg/analysis_server/tool/code_completion/metrics_util.dart
@@ -2,6 +2,8 @@
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
+import 'dart:math' as math;
+
import 'package:analysis_server/src/status/pages.dart';
import 'package:analyzer/src/generated/utilities_general.dart';
@@ -10,22 +12,28 @@
/// https://en.wikipedia.org/wiki/Average#Arithmetic_mean
class ArithmeticMeanComputer {
final String name;
- num sum = 0;
+ int sum = 0;
int count = 0;
+ int? min;
+ int? max;
ArithmeticMeanComputer(this.name);
- num get mean => sum / count;
+ double get mean => sum / count;
/// Add the data from the given [computer] to this computer.
void addData(ArithmeticMeanComputer computer) {
sum += computer.sum;
count += computer.count;
+ min = _min(min, computer.min);
+ max = _max(max, computer.max);
}
- void addValue(num val) {
+ void addValue(int val) {
sum += val;
count++;
+ min = _min(min, val);
+ max = _max(max, val);
}
void clear() {
@@ -36,8 +44,10 @@
/// Set the state of this computer to the state recorded in the decoded JSON
/// [map].
void fromJson(Map<String, dynamic> map) {
- sum = map['sum'] as num;
+ sum = map['sum'] as int;
count = map['count'] as int;
+ min = map['min'] as int?;
+ max = map['max'] as int?;
}
void printMean() {
@@ -49,8 +59,30 @@
return {
'sum': sum,
'count': count,
+ if (min != null) 'min': min,
+ if (max != null) 'max': max,
};
}
+
+ int? _max(int? first, int? second) {
+ if (first == null) {
+ return second;
+ } else if (second == null) {
+ return first;
+ } else {
+ return math.max(first, second);
+ }
+ }
+
+ int? _min(int? first, int? second) {
+ if (first == null) {
+ return second;
+ } else if (second == null) {
+ return first;
+ } else {
+ return math.min(first, second);
+ }
+ }
}
/// A simple counter class. A [String] name is passed to name the counter. Each
