Introduce and use a new class to compute the mean reciprocal rank after a run of the code completion metrics execution.
Change-Id: I72ee7fdc500d6310231b3b03a2722e63feee251a
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/133439
Reviewed-by: Brian Wilkerson <brianwilkerson@google.com>
Commit-Queue: Jaime Wren <jwren@google.com>
diff --git a/pkg/analysis_server/tool/completion_metrics/completion_metrics.dart b/pkg/analysis_server/tool/completion_metrics/completion_metrics.dart
index 88a392f..629549a 100644
--- a/pkg/analysis_server/tool/completion_metrics/completion_metrics.dart
+++ b/pkg/analysis_server/tool/completion_metrics/completion_metrics.dart
@@ -47,6 +47,7 @@
var completionKindCounter = Counter('completion kind counter');
var completionElementKindCounter =
Counter('completion element kind counter');
+ var mRRComputer = MeanReciprocalRankComputer();
print('Analyzing root: \"$_rootPath\"');
@@ -84,13 +85,20 @@
expectedCompletion.offset,
declarationsTracker);
- var fraction =
+ var place =
_placementInSuggestionList(suggestions, expectedCompletion);
- if (fraction.denominator != 0) {
+ mRRComputer.addReciprocalRank(place);
+
+ if (place.denominator != 0) {
includedCount++;
} else {
notIncludedCount++;
+
+ completionKindCounter.count(expectedCompletion.kind.toString());
+ completionElementKindCounter
+ .count(expectedCompletion.elementKind.toString());
+
if (_verbose) {
// The format "/file/path/foo.dart:3:4" makes for easier input
// with the Files dialog in IntelliJ
@@ -99,11 +107,6 @@
print(
'\tdid not include the expected completion: \"${expectedCompletion.completion}\", completion kind: ${expectedCompletion.kind.toString()}, element kind: ${expectedCompletion.elementKind.toString()}');
print('');
-
- completionKindCounter
- .count(expectedCompletion.kind.toString());
- completionElementKindCounter
- .count(expectedCompletion.elementKind.toString());
}
}
}
@@ -120,7 +123,14 @@
final percentNotIncluded = 1 - percentIncluded;
completionKindCounter.printCounterValues();
+ print('');
+
completionElementKindCounter.printCounterValues();
+ print('');
+
+ mRRComputer.printMean();
+ print('');
+
print('Summary for $_rootPath:');
print('Total number of completion tests = $totalCompletionCount');
print(
@@ -132,6 +142,7 @@
notIncludedCount = 0;
completionKindCounter.clear();
completionElementKindCounter.clear();
+ mRRComputer.clear();
}
Future<List<CompletionSuggestion>> _computeCompletionSuggestions(
diff --git a/pkg/analysis_server/tool/completion_metrics/metrics_util.dart b/pkg/analysis_server/tool/completion_metrics/metrics_util.dart
index cf7ddbe..30450de 100644
--- a/pkg/analysis_server/tool/completion_metrics/metrics_util.dart
+++ b/pkg/analysis_server/tool/completion_metrics/metrics_util.dart
@@ -52,7 +52,37 @@
print('Counts for \'$name\':');
_buckets.forEach((id, count) =>
print('[$id] $count (${printPercentage(count / _totalCount, 2)})'));
- print('');
+ }
+}
+
+/// A computer for the mean reciprocal rank,
+/// https://en.wikipedia.org/wiki/Mean_reciprocal_rank.
+class MeanReciprocalRankComputer {
+ final List<double> _ranks = [];
+ MeanReciprocalRankComputer();
+
+ double get mean {
+ double sum = 0;
+ _ranks.forEach((rank) {
+ sum += rank;
+ });
+ return rankCount == 0 ? 0 : sum / rankCount;
+ }
+
+ int get rankCount => _ranks.length;
+
+ int get ranks => _ranks.length;
+
+ void addReciprocalRank(Place place) {
+ _ranks.add(place.reciprocalRank);
+ }
+
+ void clear() => _ranks.clear();
+
+ void printMean() {
+ var mrr = mean;
+ print('Mean Reciprocal Rank = ${mrr.toStringAsFixed(5)}');
+ print('Harmonic Mean (inverse) = ${(1 / mrr).toStringAsFixed(1)}');
}
}
@@ -80,6 +110,8 @@
int get numerator => _numerator;
+ double get reciprocalRank => denominator == 0 ? 0 : numerator / denominator;
+
@override
bool operator ==(dynamic other) =>
other is Place &&
