// Copyright (c) 2020, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert';
import 'dart:io' as io;
import 'dart:math' as math;

import 'package:_fe_analyzer_shared/src/base/syntactic_entity.dart';
import 'package:analysis_server/src/domains/completion/available_suggestions.dart';
import 'package:analysis_server/src/protocol/protocol_internal.dart';
import 'package:analysis_server/src/protocol_server.dart' as protocol;
import 'package:analysis_server/src/services/completion/completion_core.dart';
import 'package:analysis_server/src/services/completion/completion_performance.dart';
import 'package:analysis_server/src/services/completion/dart/completion_manager.dart';
import 'package:analysis_server/src/services/completion/dart/feature_computer.dart';
import 'package:analysis_server/src/services/completion/dart/probability_range.dart';
import 'package:analysis_server/src/services/completion/dart/relevance_tables.g.dart';
import 'package:analysis_server/src/services/completion/dart/suggestion_builder.dart';
import 'package:analysis_server/src/services/completion/dart/utilities.dart';
import 'package:analysis_server/src/status/pages.dart';
import 'package:analyzer/dart/analysis/analysis_context_collection.dart';
import 'package:analyzer/dart/analysis/context_root.dart';
import 'package:analyzer/dart/analysis/results.dart';
import 'package:analyzer/dart/ast/ast.dart';
import 'package:analyzer/dart/ast/token.dart';
import 'package:analyzer/dart/element/element.dart'
    show
        ClassElement,
        ClassMemberElement,
        CompilationUnitElement,
        Element,
        ExecutableElement,
        ExtensionElement,
        FieldElement,
        FunctionElement,
        LocalVariableElement,
        ParameterElement,
        PrefixElement,
        TypeParameterElement,
        VariableElement;
import 'package:analyzer/diagnostic/diagnostic.dart';
import 'package:analyzer/error/error.dart' as err;
import 'package:analyzer/file_system/file_system.dart';
import 'package:analyzer/file_system/overlay_file_system.dart';
import 'package:analyzer/file_system/physical_file_system.dart';
import 'package:analyzer/src/dart/analysis/byte_store.dart';
import 'package:analyzer/src/dart/analysis/driver_based_analysis_context.dart';
import 'package:analyzer/src/dartdoc/dartdoc_directive_info.dart';
import 'package:analyzer/src/services/available_declarations.dart';
import 'package:analyzer/src/util/file_paths.dart' as file_paths;
import 'package:analyzer/src/util/performance/operation_performance.dart';
import 'package:analyzer_plugin/src/utilities/completion/optype.dart';
import 'package:args/args.dart';
import 'package:meta/meta.dart';

import 'metrics_util.dart';
import 'output_utilities.dart';
import 'relevance_table_generator.dart';
import 'visitors.dart';

Future<void> main(List<String> args) async {
  var parser = createArgParser();
  var result = parser.parse(args);

  if (!validArguments(parser, result)) {
    return io.exit(1);
  }

  var options = CompletionMetricsOptions(result);
  var provider = PhysicalResourceProvider.INSTANCE;
  if (result.wasParsed('reduceDir')) {
    var targetMetrics = <CompletionMetrics>[];
    var dir = provider.getFolder(result['reduceDir']);
    var computer = CompletionMetricsComputer('', options);
    for (var child in dir.getChildren()) {
      if (child is File) {
        var metricsList =
            (json.decode(child.readAsStringSync()) as List<dynamic>)
                .map((map) =>
                    CompletionMetrics.fromJson(map as Map<String, dynamic>))
                .toList();
        if (targetMetrics.isEmpty) {
          targetMetrics.addAll(metricsList);
        } else if (targetMetrics.length != metricsList.length) {
          throw StateError('metrics lengths differ');
        } else {
          for (var i = 0; i < targetMetrics.length; i++) {
            targetMetrics[i].addData(metricsList[i]);
          }
        }
      }
    }
    computer.targetMetrics.addAll(targetMetrics);
    computer.printResults();
    return;
  }

  var rootPath = result.rest[0];
  print('Analyzing root: "$rootPath"');
  var stopwatch = Stopwatch()..start();
  var computer = CompletionMetricsComputer(rootPath, options);
  var code = await computer.computeMetrics();
  stopwatch.stop();

  var duration = Duration(milliseconds: stopwatch.elapsedMilliseconds);
  print('');
  print('Metrics computed in $duration');

  File uniqueDataFile() {
    var dataDir = result['mapDir'];
    var baseFileName = provider.pathContext.basename(rootPath);
    var index = 1;
    while (index < 10000) {
      var suffix = (index++).toString();
      suffix = '0000'.substring(suffix.length) + suffix + '.json';
      var fileName = baseFileName + suffix;
      var filePath = provider.pathContext.join(dataDir, fileName);
      var file = provider.getFile(filePath);
      if (!file.exists) {
        return file;
      }
    }

    /// If there are more than 10000 directories with the same name, just
    /// overwrite a previously generated file.
    var fileName = baseFileName + '9999';
    var filePath = provider.pathContext.join(dataDir, fileName);
    return provider.getFile(filePath);
  }

  if (result.wasParsed('mapDir')) {
    var dataFile = uniqueDataFile();
    var map =
        computer.targetMetrics.map((metrics) => metrics.toJson()).toList();
    dataFile.writeAsStringSync(json.encode(map));
  } else {
    computer.printResults();
  }
  return io.exit(code);
}

/// A [Counter] to track the performance of each of the completion strategies
/// that are being compared.
Counter rankComparison = Counter('relevance rank comparison');

/// Create a parser that can be used to parse the command-line arguments.
ArgParser createArgParser() {
  return ArgParser()
    ..addFlag(CompletionMetricsOptions.AVAILABLE_SUGGESTIONS,
        abbr: 'a',
        help:
            'Use the available suggestions feature in the Analysis Server when '
            'computing the set of code completions. With this feature enabled, '
            'completion will match the support in the Dart Plugin for '
            'IntelliJ, without this enabled the completion support matches the '
            'support in LSP.',
        defaultsTo: false,
        negatable: false)
    ..addFlag(CompletionMetricsOptions.MD,
        help: 'Use markdown as the output format.',
        defaultsTo: false,
        negatable: false)
    ..addOption(
      'help',
      abbr: 'h',
      help: 'Print this help message.',
    )
    ..addOption(CompletionMetricsOptions.OVERLAY,
        allowed: [
          CompletionMetricsOptions.OVERLAY_NONE,
          CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN,
          CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE
        ],
        defaultsTo: CompletionMetricsOptions.OVERLAY_NONE,
        help:
            'Before attempting a completion at the location of each token, the '
            'token can be removed, or the rest of the file can be removed to '
            'test code completion with diverse methods. The default mode is to '
            'complete at the start of the token without modifying the file.')
    ..addFlag(CompletionMetricsOptions.PRINT_MISSED_COMPLETION_DETAILS,
        defaultsTo: false,
        help:
            'Print detailed information every time a completion request fails '
            'to produce a suggestions matching the expected suggestion.',
        negatable: false)
    ..addFlag(CompletionMetricsOptions.PRINT_MISSED_COMPLETION_SUMMARY,
        defaultsTo: false,
        help: 'Print summary information about the times that a completion '
            'request failed to produce a suggestions matching the expected '
            'suggestion.',
        negatable: false)
    ..addFlag(CompletionMetricsOptions.PRINT_MISSING_INFORMATION,
        defaultsTo: false,
        help: 'Print information about places where no completion location was '
            'computed and about information that is missing in the completion '
            'tables.',
        negatable: false)
    ..addFlag(CompletionMetricsOptions.PRINT_MRR_BY_LOCATION,
        defaultsTo: false,
        help:
            'Print information about the mrr score achieved at each completion '
            'location. This can help focus efforts to improve the overall '
            'score by pointing out the locations that are causing the biggest '
            'impact.',
        negatable: false)
    ..addFlag(CompletionMetricsOptions.PRINT_SLOWEST_RESULTS,
        defaultsTo: false,
        help: 'Print information about the completion requests that were the '
            'slowest to return suggestions.',
        negatable: false)
    ..addFlag(CompletionMetricsOptions.PRINT_WORST_RESULTS,
        defaultsTo: false,
        help: 'Print information about the completion requests that had the '
            'worst mrr scores.',
        negatable: false)
    ..addOption(
      'mapDir',
      help: 'The absolute path of the directory to which the completion '
          'metrics data will be written. Using this option will prevent the '
          'completion results from being written in a textual form.',
    )
    ..addOption(
      'reduceDir',
      help: 'The absolute path of the directory from which the completion '
          'metrics data will be read.',
    );
}

/// Print usage information for this tool.
void printUsage(ArgParser parser, {String error}) {
  if (error != null) {
    print(error);
    print('');
  }
  print('usage: dart completion_metrics.dart [options] packagePath');
  print('');
  print('Compute code completion health metrics.');
  print('');
  print(parser.usage);
}

/// Return `true` if the command-line arguments (represented by the [result] and
/// parsed by the [parser]) are valid.
bool validArguments(ArgParser parser, ArgResults result) {
  if (result.wasParsed('help')) {
    printUsage(parser);
    return false;
  } else if (result.wasParsed('reduceDir')) {
    return validateDir(parser, result['reduceDir']);
  } else if (result.rest.length != 1) {
    printUsage(parser, error: 'No package path specified.');
    return false;
  }
  if (result.wasParsed('mapDir')) {
    return validateDir(parser, result['mapDir']);
  }
  return validateDir(parser, result.rest[0]);
}

/// An indication of the group in which the completion falls for the purposes of
/// subdividing the results.
enum CompletionGroup {
  classElement,
  constructorElement,
  enumElement,
  extensionElement,

  /// An instance member of a class, enum, mixin or extension.
  instanceMember,

  labelElement,
  localFunctionElement,
  localVariableElement,
  mixinElement,
  parameterElement,
  prefixElement,

  /// A static member of a class, enum, mixin or extension.
  staticMember,

  topLevelMember,
  typeParameterElement,

  // Groups for keywords.

  keywordDynamic,
  keywordVoid,

  /// Anything that doesn't fit in one of the other groups.
  unknown,
}

/// A wrapper for the collection of [Counter] and [MeanReciprocalRankComputer]
/// objects for a run of [CompletionMetricsComputer].
class CompletionMetrics {
  /// The maximum number of slowest results to collect.
  static const maxSlowestResults = 5;

  /// The maximum number of worst results to collect.
  static const maxWorstResults = 5;

  /// The name associated with this set of metrics.
  final String name;

  /// The function to be executed when this metrics collector is enabled.
  final void Function() enableFunction;

  /// The function to be executed when this metrics collector is disabled.
  final void Function() disableFunction;

  final Counter completionCounter = Counter('all completions');

  final Counter completionMissedTokenCounter =
      Counter('unsuccessful completion token counter');

  final Counter completionKindCounter =
      Counter('unsuccessful completion kind counter');

  final Counter completionElementKindCounter =
      Counter('unsuccessful completion element kind counter');

  final ArithmeticMeanComputer meanCompletionMS =
      ArithmeticMeanComputer('ms per completion');

  final MeanReciprocalRankComputer mrrComputer =
      MeanReciprocalRankComputer('all completions');

  final MeanReciprocalRankComputer successfulMrrComputer =
      MeanReciprocalRankComputer('successful completions');

  /// A table mapping completion groups to the mrr computer used to track the
  /// quality of suggestions for those groups.
  final Map<CompletionGroup, MeanReciprocalRankComputer> groupMrrComputers = {};

  /// A table mapping locations to the mrr computer used to track the quality of
  /// suggestions for those locations.
  final Map<String, MeanReciprocalRankComputer> locationMrrComputers = {};

  final ArithmeticMeanComputer charsBeforeTop =
      ArithmeticMeanComputer('chars_before_top');

  final ArithmeticMeanComputer charsBeforeTopFive =
      ArithmeticMeanComputer('chars_before_top_five');

  final ArithmeticMeanComputer insertionLengthTheoretical =
      ArithmeticMeanComputer('insertion_length_theoretical');

  /// The places in which a completion location was requested when none was
  /// available.
  final Set<String> missingCompletionLocations = {};

  /// The completion locations for which no relevance table was available.
  final Set<String> missingCompletionLocationTables = {};

  final Map<CompletionGroup, List<CompletionResult>> slowestResults = {};

  final Map<CompletionGroup, List<CompletionResult>> worstResults = {};

  CompletionMetrics(this.name, {this.enableFunction, this.disableFunction});

  /// Return an instance extracted from the decoded JSON [map].
  factory CompletionMetrics.fromJson(Map<String, dynamic> map) {
    var metrics = CompletionMetrics(map['name'] as String);
    metrics.completionCounter
        .fromJson(map['completionCounter'] as Map<String, dynamic>);
    metrics.completionMissedTokenCounter
        .fromJson(map['completionMissedTokenCounter'] as Map<String, dynamic>);
    metrics.completionKindCounter
        .fromJson(map['completionKindCounter'] as Map<String, dynamic>);
    metrics.completionElementKindCounter
        .fromJson(map['completionElementKindCounter'] as Map<String, dynamic>);
    metrics.meanCompletionMS
        .fromJson(map['meanCompletionMS'] as Map<String, dynamic>);
    metrics.mrrComputer.fromJson(map['mrrComputer'] as Map<String, dynamic>);
    metrics.successfulMrrComputer
        .fromJson(map['successfulMrrComputer'] as Map<String, dynamic>);
    for (var entry
        in (map['groupMrrComputers'] as Map<String, dynamic>).entries) {
      var group = CompletionGroup.values[int.parse(entry.key)];
      metrics.groupMrrComputers[group] = MeanReciprocalRankComputer(group.name)
        ..fromJson(entry.value);
    }
    for (var entry
        in (map['locationMrrComputers'] as Map<String, dynamic>).entries) {
      var location = entry.key;
      metrics.locationMrrComputers[location] =
          MeanReciprocalRankComputer(location)..fromJson(entry.value);
    }
    metrics.charsBeforeTop
        .fromJson(map['charsBeforeTop'] as Map<String, dynamic>);
    metrics.charsBeforeTopFive
        .fromJson(map['charsBeforeTopFive'] as Map<String, dynamic>);
    metrics.insertionLengthTheoretical
        .fromJson(map['insertionLengthTheoretical'] as Map<String, dynamic>);
    for (var element in map['missingCompletionLocations'] as List<dynamic>) {
      metrics.missingCompletionLocations.add(element);
    }
    for (var element
        in map['missingCompletionLocationTables'] as List<dynamic>) {
      metrics.missingCompletionLocationTables.add(element);
    }
    for (var entry in (map['slowestResults'] as Map<String, dynamic>).entries) {
      var group = CompletionGroup.values[int.parse(entry.key)];
      var results = (entry.value as List<dynamic>)
          .map((map) => CompletionResult.fromJson(map as Map<String, dynamic>))
          .toList();
      metrics.slowestResults[group] = results;
    }
    for (var entry in (map['worstResults'] as Map<String, dynamic>).entries) {
      var group = CompletionGroup.values[int.parse(entry.key)];
      var results = (entry.value as List<dynamic>)
          .map((map) => CompletionResult.fromJson(map as Map<String, dynamic>))
          .toList();
      metrics.worstResults[group] = results;
    }
    return metrics;
  }

  /// Add the data from the given [metrics] to this metrics.
  void addData(CompletionMetrics metrics) {
    completionCounter.addData(metrics.completionCounter);
    completionMissedTokenCounter.addData(metrics.completionMissedTokenCounter);
    completionKindCounter.addData(metrics.completionKindCounter);
    completionElementKindCounter.addData(metrics.completionElementKindCounter);
    meanCompletionMS.addData(metrics.meanCompletionMS);
    mrrComputer.addData(metrics.mrrComputer);
    successfulMrrComputer.addData(metrics.successfulMrrComputer);
    for (var entry in metrics.groupMrrComputers.entries) {
      var group = entry.key;
      groupMrrComputers
          .putIfAbsent(group, () => MeanReciprocalRankComputer(group.name))
          .addData(entry.value);
    }
    for (var entry in metrics.locationMrrComputers.entries) {
      var location = entry.key;
      locationMrrComputers
          .putIfAbsent(location, () => MeanReciprocalRankComputer(location))
          .addData(entry.value);
    }
    charsBeforeTop.addData(metrics.charsBeforeTop);
    charsBeforeTopFive.addData(metrics.charsBeforeTopFive);
    insertionLengthTheoretical.addData(metrics.insertionLengthTheoretical);
    missingCompletionLocations.addAll(metrics.missingCompletionLocations);
    missingCompletionLocationTables
        .addAll(metrics.missingCompletionLocationTables);
    for (var resultList in metrics.slowestResults.values) {
      for (var result in resultList) {
        _recordSlowestResult(result);
      }
    }
    for (var resultList in metrics.worstResults.values) {
      for (var result in resultList) {
        _recordWorstResult(result);
      }
    }
  }

  /// Perform any operations required in order to revert computing the kind of
  /// completions represented by this metrics collector.
  void disable() {
    if (disableFunction != null) {
      disableFunction();
    }
  }

  /// Perform any initialization required in order to compute the kind of
  /// completions represented by this metrics collector.
  void enable() {
    if (enableFunction != null) {
      enableFunction();
    }
  }

  /// Record the completion [result]. This method handles the worst ranked items
  /// as well as the longest sets of results to compute.
  void recordCompletionResult(
      CompletionResult result, MetricsSuggestionListener listener) {
    _recordTime(result);
    _recordMrr(result);
    _recordWorstResult(result);
    _recordSlowestResult(result);
    _recordMissingInformation(listener);
  }

  Map<String, dynamic> toJson() {
    return {
      'name': name,
      'completionCounter': completionCounter.toJson(),
      'completionMissedTokenCounter': completionMissedTokenCounter.toJson(),
      'completionKindCounter': completionKindCounter.toJson(),
      'completionElementKindCounter': completionElementKindCounter.toJson(),
      'meanCompletionMS': meanCompletionMS.toJson(),
      'mrrComputer': mrrComputer.toJson(),
      'successfulMrrComputer': successfulMrrComputer.toJson(),
      'groupMrrComputers': groupMrrComputers
          .map((key, value) => MapEntry(key.index.toString(), value.toJson())),
      'locationMrrComputers': locationMrrComputers
          .map((key, value) => MapEntry(key, value.toJson())),
      'charsBeforeTop': charsBeforeTop.toJson(),
      'charsBeforeTopFive': charsBeforeTopFive.toJson(),
      'insertionLengthTheoretical': insertionLengthTheoretical.toJson(),
      'missingCompletionLocations': missingCompletionLocations.toList(),
      'missingCompletionLocationTables':
          missingCompletionLocationTables.toList(),
      'slowestResults': slowestResults.map((key, value) => MapEntry(
          key.index.toString(),
          value.map((result) => result.toJson()).toList())),
      'worstResults': worstResults.map((key, value) => MapEntry(
          key.index.toString(),
          value.map((result) => result.toJson()).toList())),
    };
  }

  /// If the completion location was requested but missing when computing the
  /// [result], then record where that happened.
  void _recordMissingInformation(MetricsSuggestionListener listener) {
    var location = listener?.missingCompletionLocation;
    if (location != null) {
      missingCompletionLocations.add(location);
    } else {
      location = listener?.missingCompletionLocationTable;
      if (location != null) {
        missingCompletionLocationTables.add(location);
      }
    }
  }

  /// Record the MRR for the [result].
  void _recordMrr(CompletionResult result) {
    var rank = result.place.rank;
    // Record globally.
    successfulMrrComputer.addRank(rank);
    // Record by group.
    var group = result.group;
    groupMrrComputers
        .putIfAbsent(group, () => MeanReciprocalRankComputer(group.name))
        .addRank(rank);
    // Record by completion location.
    var location = result.completionLocation;
    if (location != null) {
      var computer = locationMrrComputers.putIfAbsent(
          location, () => MeanReciprocalRankComputer(location));
      computer.addRank(rank);
    }
  }

  /// If the [result] is took longer than any previously recorded results,
  /// record it.
  void _recordSlowestResult(CompletionResult result) {
    var results = slowestResults.putIfAbsent(result.group, () => []);
    if (results.length >= maxSlowestResults) {
      if (result.elapsedMS <= results.last.elapsedMS) {
        return;
      }
      results.removeLast();
    }
    results.add(result);
    results.sort((first, second) => second.elapsedMS - first.elapsedMS);
  }

  /// Record this elapsed ms count for the average ms count.
  void _recordTime(CompletionResult result) {
    meanCompletionMS.addValue(result.elapsedMS);
  }

  /// If the [result] is worse than any previously recorded results, record it.
  void _recordWorstResult(CompletionResult result) {
    var results = worstResults.putIfAbsent(result.group, () => []);
    if (results.length >= maxWorstResults) {
      if (result.place.rank <= results.last.place.rank) {
        return;
      }
      results.removeLast();
    }
    results.add(result);
    results.sort((first, second) => second.place.rank - first.place.rank);
  }
}

/// This is the main metrics computer class for code completions. After the
/// object is constructed, [computeCompletionMetrics] is executed to do analysis
/// and print a summary of the metrics gathered from the completion tests.
class CompletionMetricsComputer {
  final String rootPath;

  final CompletionMetricsOptions options;

  ResolvedUnitResult _resolvedUnitResult;

  /// The int to be returned from the [computeMetrics] call.
  int resultCode;

  /// A list of the metrics to be computed.
  final List<CompletionMetrics> targetMetrics = [];

  final OverlayResourceProvider _provider =
      OverlayResourceProvider(PhysicalResourceProvider.INSTANCE);

  int overlayModificationStamp = 0;

  CompletionMetricsComputer(this.rootPath, this.options);

  /// Compare the metrics when each feature is used in isolation.
  void compareIndividualFeatures() {
    var featureNames = FeatureComputer.featureNames;
    var featureCount = featureNames.length;
    for (var i = 0; i < featureCount; i++) {
      var weights = List.filled(featureCount, 0.00);
      weights[i] = 1.00;
      targetMetrics.add(CompletionMetrics(featureNames[i], enableFunction: () {
        FeatureComputer.featureWeights = weights;
      }, disableFunction: () {
        FeatureComputer.featureWeights = FeatureComputer.defaultFeatureWeights;
      }));
    }
  }

  /// Compare the relevance [tables] to the default relevance tables.
  void compareRelevanceTables(List<RelevanceTables> tables) {
    assert(tables.isNotEmpty);
    for (var tablePair in tables) {
      targetMetrics.add(CompletionMetrics(tablePair.name, enableFunction: () {
        elementKindRelevance = tablePair.elementKindRelevance;
        keywordRelevance = tablePair.keywordRelevance;
      }, disableFunction: () {
        elementKindRelevance = defaultElementKindRelevance;
        keywordRelevance = defaultKeywordRelevance;
      }));
    }
  }

  Future<int> computeMetrics() async {
    resultCode = 0;
    // To compare two or more changes to completions, add a `CompletionMetrics`
    // object with enable and disable functions to the list of `targetMetrics`.
    targetMetrics.add(CompletionMetrics('shipping',
        enableFunction: null, disableFunction: null));

    // To compare two or more relevance tables, uncomment the line below and
    // add the `RelevanceTables` to the list. The default relevance tables
    // should not be included in the list.
//     compareRelevanceTables([]);

    // To compare the relative benefit from each of the features, uncomment the
    // line below.
//    compareIndividualFeatures();

    final collection = AnalysisContextCollection(
      includedPaths: [rootPath],
      resourceProvider: PhysicalResourceProvider.INSTANCE,
    );
    for (var context in collection.contexts) {
      await _computeInContext(context.contextRoot);
    }
    return resultCode;
  }

  int forEachExpectedCompletion(
      CompletionRequestImpl request,
      MetricsSuggestionListener listener,
      ExpectedCompletion expectedCompletion,
      String completionLocation,
      List<protocol.CompletionSuggestion> suggestions,
      CompletionMetrics metrics,
      int elapsedMS) {
    assert(suggestions != null);

    var place = placementInSuggestionList(suggestions, expectedCompletion);

    metrics.mrrComputer.addRank(place.rank);

    if (place.denominator != 0) {
      metrics.completionCounter.count('successful');

      var rank = place.rank;
      var suggestion = suggestions[rank - 1];
      var actualSuggestion =
          SuggestionData(suggestion, listener.featureMap[suggestion]);
      List<SuggestionData> topSuggestions;
      Map<int, int> precedingRelevanceCounts;
      if (options.printWorstResults) {
        topSuggestions = suggestions
            .sublist(0, math.min(10, suggestions.length))
            .map((suggestion) =>
                SuggestionData(suggestion, listener.featureMap[suggestion]))
            .toList();
        precedingRelevanceCounts = <int, int>{};
        for (var i = 0; i < rank - 1; i++) {
          var relevance = suggestions[i].relevance;
          precedingRelevanceCounts[relevance] =
              (precedingRelevanceCounts[relevance] ?? 0) + 1;
        }
      }
      metrics.recordCompletionResult(
          CompletionResult(
              place,
              request,
              actualSuggestion,
              topSuggestions,
              precedingRelevanceCounts,
              expectedCompletion,
              completionLocation,
              elapsedMS),
          listener);

      var charsBeforeTop =
          _computeCharsBeforeTop(expectedCompletion, suggestions);
      metrics.charsBeforeTop.addValue(charsBeforeTop);
      metrics.charsBeforeTopFive.addValue(
          _computeCharsBeforeTop(expectedCompletion, suggestions, minRank: 5));
      metrics.insertionLengthTheoretical
          .addValue(expectedCompletion.completion.length - charsBeforeTop);

      return place.rank;
    } else {
      metrics.completionCounter.count('unsuccessful');

      metrics.completionMissedTokenCounter.count(expectedCompletion.completion);
      metrics.completionKindCounter.count(expectedCompletion.kind.toString());
      metrics.completionElementKindCounter
          .count(expectedCompletion.elementKind.toString());

      if (options.printMissedCompletionDetails) {
        protocol.CompletionSuggestion closeMatchSuggestion;
        for (var suggestion in suggestions) {
          if (suggestion.completion == expectedCompletion.completion) {
            closeMatchSuggestion = suggestion;
          }
        }

        print('missing completion (${metrics.name}):');
        print('$expectedCompletion');
        if (closeMatchSuggestion != null) {
          print('    close matching completion that was in the list:');
          print('    $closeMatchSuggestion');
        }
        print('');
      }

      return -1;
    }
  }

  void printComparisons() {
    printHeading(1, 'Comparison of experiments');
    printMrrComparison();
    printCounter(rankComparison);
    printOtherMetrics();
    printCompletionCounts();
  }

  void printCompletionCounts() {
    String toString(int count, int totalCount) {
      return '$count (${printPercentage(count / totalCount, 2)})';
    }

    var counters = targetMetrics.map((metrics) => metrics.completionCounter);
    var table = [
      ['', for (var metrics in targetMetrics) metrics.name],
      ['total', for (var counter in counters) counter.totalCount.toString()],
      [
        'successful',
        for (var counter in counters)
          toString(counter.getCountOf('successful'), counter.totalCount)
      ],
      [
        'unsuccessful',
        for (var counter in counters)
          toString(counter.getCountOf('unsuccessful'), counter.totalCount)
      ],
    ];
    rightJustifyColumns(table, range(1, table[0].length));

    printHeading(2, 'Comparison of completion counts');
    printTable(table);
  }

  void printCounter(Counter counter) {
    var name = counter.name;
    var total = counter.totalCount;
    printHeading(2, "Counts for '$name' (total = $total)");
    counter.printCounterValues();
  }

  void printHeading(int level, String heading) {
    if (options.markdown) {
      var prefix = '#' * level;
      print('$prefix $heading');
    } else {
      print(heading);
      print(((level == 1) ? '=' : '-') * heading.length);
    }
    print('');
  }

  void printMetrics(CompletionMetrics metrics) {
    printHeading(1, 'Completion metrics for ${metrics.name}');
    if (options.printMissedCompletionSummary) {
      printCounter(metrics.completionMissedTokenCounter);
      printCounter(metrics.completionKindCounter);
      printCounter(metrics.completionElementKindCounter);
    }

    List<String> toRow(MeanReciprocalRankComputer computer) {
      return [
        computer.name,
        computer.mrr.toStringAsFixed(6),
        (1 / computer.mrr).toStringAsFixed(3),
        computer.mrr_5.toStringAsFixed(6),
        (1 / computer.mrr_5).toStringAsFixed(3),
        computer.count.toString(),
      ];
    }

    var groups = metrics.groupMrrComputers.keys.toList();
    groups.sort((first, second) => first.name.compareTo(second.name));
    var table = [
      ['', 'mrr', 'inverse mrr', 'mrr_5', 'inverse mrr_5', 'count'],
      toRow(metrics.mrrComputer),
      toRow(metrics.successfulMrrComputer),
      for (var group in groups) toRow(metrics.groupMrrComputers[group]),
    ];
    rightJustifyColumns(table, [2, 4, 5]);

    printHeading(2, 'Mean Reciprocal Rank');
    printTable(table);

    if (options.printMrrByLocation) {
      var lines = <LocationTableLine>[];
      for (var entry in metrics.locationMrrComputers.entries) {
        var count = entry.value.count;
        var mrr = (1 / entry.value.mrr);
        var mrr_5 = (1 / entry.value.mrr_5);
        var product = count * mrr;
        lines.add(LocationTableLine(
            label: entry.key,
            product: product,
            count: count,
            mrr: mrr,
            mrr_5: mrr_5));
      }
      lines.sort((first, second) => second.product.compareTo(first.product));
      var table = <List<String>>[];
      table.add(['Location', 'Product', 'Count', 'Mrr', 'Mrr_5']);
      for (var line in lines) {
        var location = line.label;
        var product = line.product.truncate().toString();
        var count = line.count.toString();
        var mrr = line.mrr.toStringAsFixed(3);
        var mrr_5 = line.mrr_5.toStringAsFixed(3);
        table.add([location, product, count, mrr, mrr_5]);
      }
      printTable(table);
    }
  }

  void printMissingInformation(CompletionMetrics metrics) {
    var locations = metrics.missingCompletionLocations;
    if (locations.isNotEmpty) {
      print('');
      printHeading(2, 'Missing completion location in the following places');
      for (var location in locations.toList()..sort()) {
        print('- $location');
      }
    }

    var tables = metrics.missingCompletionLocationTables;
    if (tables.isNotEmpty) {
      print('');
      printHeading(2, 'Missing tables for the following completion locations');
      for (var table in tables.toList()..sort()) {
        print('- $table');
      }
    }
  }

  void printMrrComparison() {
    List<String> toRow(Iterable<MeanReciprocalRankComputer> sources) {
      var computers = sources.toList();
      var baseComputer = computers.first;
      var row = [baseComputer.name];
      var baseInverseMrr = 1 / baseComputer.mrr;
      row.add(baseInverseMrr.toStringAsFixed(3));
      for (var i = 1; i < computers.length; i++) {
        var inverseMrr = 1 / computers[i].mrr;
        var delta = inverseMrr - baseInverseMrr;
        row.add('|');
        row.add(inverseMrr.toStringAsFixed(3));
        row.add(delta.toStringAsFixed(3));
      }
      return row;
    }

    var columnHeaders = [' ', targetMetrics[0].name];
    for (var i = 1; i < targetMetrics.length; i++) {
      columnHeaders.add('|');
      columnHeaders.add('${targetMetrics[i].name}');
      columnHeaders.add('delta');
    }
    var blankRow = [for (int i = 0; i < columnHeaders.length; i++) ''];
    var table = [
      columnHeaders,
      toRow(targetMetrics.map((metrics) => metrics.mrrComputer)),
      toRow(targetMetrics.map((metrics) => metrics.successfulMrrComputer)),
      blankRow,
    ];
    var elementKinds = targetMetrics
        .expand((metrics) => metrics.groupMrrComputers.keys)
        .toSet()
        .toList();
    elementKinds.sort((first, second) => first.name.compareTo(second.name));
    for (var kind in elementKinds) {
      table.add(toRow(
          targetMetrics.map((metrics) => metrics.groupMrrComputers[kind])));
    }
    if (options.printMrrByLocation) {
      table.add(blankRow);
      var locations = targetMetrics
          .expand((metrics) => metrics.locationMrrComputers.keys)
          .toSet()
          .toList();
      locations.sort();
      for (var location in locations) {
        table.add(toRow(targetMetrics
            .map((metrics) => metrics.locationMrrComputers[location])));
      }
    }
    rightJustifyColumns(table, range(1, table[0].length));

    printHeading(2, 'Comparison of inverse mean reciprocal ranks');
    print('A lower value is better, so a negative delta is good.');
    print('');
    printTable(table);
  }

  void printOtherMetrics() {
    List<String> toRow(Iterable<ArithmeticMeanComputer> sources) {
      var computers = sources.toList();
      var row = [computers.first.name];
      for (var computer in computers) {
        row.add(computer.mean.toStringAsFixed(6));
      }
      return row;
    }

    var table = [
      ['', for (var metrics in targetMetrics) metrics.name],
      toRow(targetMetrics.map((metrics) => metrics.meanCompletionMS)),
      toRow(targetMetrics.map((metrics) => metrics.charsBeforeTop)),
      toRow(targetMetrics.map((metrics) => metrics.charsBeforeTopFive)),
      toRow(targetMetrics.map((metrics) => metrics.insertionLengthTheoretical)),
    ];
    rightJustifyColumns(table, range(1, table[0].length));

    printHeading(2, 'Comparison of other metrics');
    printTable(table);
  }

  void printResults() {
    if (targetMetrics.length > 1) {
      print('');
      printComparisons();
    }
    var needsBlankLine = false;
    for (var metrics in targetMetrics) {
      if (needsBlankLine) {
        print('');
      } else {
        needsBlankLine = true;
      }
      printMetrics(metrics);

      if (options.printMissingInformation) {
        printMissingInformation(metrics);
      }
      if (options.printSlowestResults) {
        printSlowestResults(metrics);
      }
      if (options.printWorstResults) {
        printWorstResults(metrics);
      }
    }
  }

  void printSlowestResults(CompletionMetrics metrics) {
    var slowestResults = metrics.slowestResults;
    var groups = slowestResults.keys.toList();
    groups.sort((first, second) => first.name.compareTo(second.name));
    print('');
    printHeading(2, 'The slowest completion results to compute');
    for (var group in groups) {
      _printSlowestResults('In ${group.name}', slowestResults[group]);
    }
  }

  void printWorstResults(CompletionMetrics metrics) {
    var worstResults = metrics.worstResults;
    var groups = worstResults.keys.toList();
    groups.sort((first, second) => first.name.compareTo(second.name));
    print('');
    printHeading(2, 'The worst completion results');
    for (var group in groups) {
      _printWorstResults('In ${group.name}', worstResults[group]);
    }
  }

  int _computeCharsBeforeTop(ExpectedCompletion target,
      List<protocol.CompletionSuggestion> suggestions,
      {int minRank = 1}) {
    var rank = placementInSuggestionList(suggestions, target).rank;
    if (rank <= minRank) {
      return 0;
    }
    var expected = target.completion;
    for (var i = 1; i < expected.length + 1; i++) {
      var prefix = expected.substring(0, i);
      var filteredSuggestions = _filterSuggestions(prefix, suggestions);
      rank = placementInSuggestionList(filteredSuggestions, target).rank;
      if (rank <= minRank) {
        return i;
      }
    }
    return expected.length;
  }

  Future<List<protocol.CompletionSuggestion>> _computeCompletionSuggestions(
      MetricsSuggestionListener listener,
      OperationPerformanceImpl performance,
      CompletionRequestImpl request,
      [DeclarationsTracker declarationsTracker,
      protocol.CompletionAvailableSuggestionsParams
          availableSuggestionsParams]) async {
    List<protocol.CompletionSuggestion> suggestions;

    if (declarationsTracker == null) {
      // available suggestions == false
      suggestions = await DartCompletionManager(
        dartdocDirectiveInfo: DartdocDirectiveInfo(),
        listener: listener,
      ).computeSuggestions(performance, request);
    } else {
      // available suggestions == true
      var includedElementKinds = <protocol.ElementKind>{};
      var includedElementNames = <String>{};
      var includedSuggestionRelevanceTagList =
          <protocol.IncludedSuggestionRelevanceTag>[];
      var includedSuggestionSetList = <protocol.IncludedSuggestionSet>[];
      suggestions = await DartCompletionManager(
        dartdocDirectiveInfo: DartdocDirectiveInfo(),
        includedElementKinds: includedElementKinds,
        includedElementNames: includedElementNames,
        includedSuggestionRelevanceTags: includedSuggestionRelevanceTagList,
        listener: listener,
      ).computeSuggestions(performance, request);

      computeIncludedSetList(declarationsTracker, request.result,
          includedSuggestionSetList, includedElementNames);

      var includedSuggestionSetMap = {
        for (var includedSuggestionSet in includedSuggestionSetList)
          includedSuggestionSet.id: includedSuggestionSet,
      };

      var includedSuggestionRelevanceTagMap = {
        for (var includedSuggestionRelevanceTag
            in includedSuggestionRelevanceTagList)
          includedSuggestionRelevanceTag.tag:
              includedSuggestionRelevanceTag.relevanceBoost,
      };

      for (var availableSuggestionSet
          in availableSuggestionsParams.changedLibraries) {
        var id = availableSuggestionSet.id;
        for (var availableSuggestion in availableSuggestionSet.items) {
          // Exclude available suggestions where this element kind doesn't match
          // an element kind in includedElementKinds.
          var elementKind = availableSuggestion.element?.kind;
          if (elementKind != null &&
              includedElementKinds.contains(elementKind)) {
            if (includedSuggestionSetMap.containsKey(id)) {
              var relevance = includedSuggestionSetMap[id].relevance;

              // Search for any matching relevance tags to apply any boosts
              if (includedSuggestionRelevanceTagList.isNotEmpty &&
                  availableSuggestion.relevanceTags != null &&
                  availableSuggestion.relevanceTags.isNotEmpty) {
                for (var tag in availableSuggestion.relevanceTags) {
                  if (includedSuggestionRelevanceTagMap.containsKey(tag)) {
                    // apply the boost
                    relevance += includedSuggestionRelevanceTagMap[tag];
                  }
                }
              }
              suggestions
                  .add(availableSuggestion.toCompletionSuggestion(relevance));
            }
          }
        }
      }
    }

    suggestions.sort(completionComparator);
    return suggestions;
  }

  /// Compute the metrics for the files in the context [root], creating a
  /// separate context collection to prevent accumulating memory. The metrics
  /// should be captured in the [collector].
  Future<void> _computeInContext(ContextRoot root) async {
    // Create a new collection to avoid consuming large quantities of memory.
    final collection = AnalysisContextCollection(
      includedPaths: root.includedPaths.toList(),
      excludedPaths: root.excludedPaths.toList(),
      resourceProvider: _provider,
    );

    var context = collection.contexts[0];

    // Set the DeclarationsTracker, only call doWork to build up the available
    // suggestions if doComputeCompletionsFromAnalysisServer is true.
    DeclarationsTracker declarationsTracker;
    protocol.CompletionAvailableSuggestionsParams availableSuggestionsParams;
    if (options.availableSuggestions) {
      declarationsTracker = DeclarationsTracker(
          MemoryByteStore(), PhysicalResourceProvider.INSTANCE);
      declarationsTracker.addContext(context);
      while (declarationsTracker.hasWork) {
        declarationsTracker.doWork();
      }

      // Have the AvailableDeclarationsSet computed to use later.
      availableSuggestionsParams = createCompletionAvailableSuggestions(
          declarationsTracker.allLibraries.toList(), []);

      // assert that this object is not null, throw if it is.
      if (availableSuggestionsParams == null) {
        throw Exception('availableSuggestionsParam not computable.');
      }
    }

    // Loop through each file, resolve the file and call
    // forEachExpectedCompletion
    var pathContext = context.contextRoot.resourceProvider.pathContext;
    for (var filePath in context.contextRoot.analyzedFiles()) {
      if (file_paths.isDart(pathContext, filePath)) {
        try {
          _resolvedUnitResult =
              await context.currentSession.getResolvedUnit(filePath);

          var analysisError = getFirstErrorOrNull(_resolvedUnitResult);
          if (analysisError != null) {
            print('File $filePath skipped due to errors such as:');
            print('  ${analysisError.toString()}');
            print('');
            resultCode = 1;
            continue;
          }

          // Use the ExpectedCompletionsVisitor to compute the set of expected
          // completions for this CompilationUnit.
          final visitor = ExpectedCompletionsVisitor(filePath);
          _resolvedUnitResult.unit.accept(visitor);

          for (var expectedCompletion in visitor.expectedCompletions) {
            var resolvedUnitResult = _resolvedUnitResult;

            // If an overlay option is being used, compute the overlay file, and
            // have the context reanalyze the file
            if (options.overlay != CompletionMetricsOptions.OVERLAY_NONE) {
              var overlayContents = _getOverlayContents(
                  _resolvedUnitResult.content, expectedCompletion);

              _provider.setOverlay(filePath,
                  content: overlayContents,
                  modificationStamp: overlayModificationStamp++);
              (context as DriverBasedAnalysisContext)
                  .driver
                  .changeFile(filePath);
              resolvedUnitResult =
                  await context.currentSession.getResolvedUnit(filePath);
            }

            // As this point the completion suggestions are computed,
            // and results are collected with varying settings for
            // comparison:

            Future<int> handleExpectedCompletion(
                {MetricsSuggestionListener listener,
                @required CompletionMetrics metrics}) async {
              var stopwatch = Stopwatch()..start();
              var request = CompletionRequestImpl(
                resolvedUnitResult,
                expectedCompletion.offset,
                CompletionPerformance(),
              );
              var directiveInfo = DartdocDirectiveInfo();

              OpType opType;
              List<protocol.CompletionSuggestion> suggestions;
              await request.performance.runRequestOperation(
                (performance) async {
                  var dartRequest = await DartCompletionRequestImpl.from(
                      performance, request, directiveInfo);
                  opType =
                      OpType.forCompletion(dartRequest.target, request.offset);
                  suggestions = await _computeCompletionSuggestions(
                    listener,
                    performance,
                    request,
                    declarationsTracker,
                    availableSuggestionsParams,
                  );
                },
              );
              stopwatch.stop();

              return forEachExpectedCompletion(
                  request,
                  listener,
                  expectedCompletion,
                  opType.completionLocation,
                  suggestions,
                  metrics,
                  stopwatch.elapsedMilliseconds);
            }

            var bestRank = -1;
            var bestName = '';
            for (var metrics in targetMetrics) {
              // Compute the completions.
              metrics.enable();
              // if (FeatureComputer.noDisabledFeatures) {
              //   var line = expectedCompletion.lineNumber;
              //   var column = expectedCompletion.columnNumber;
              //   print('$filePath:$line:$column');
              // }
              var listener = MetricsSuggestionListener();
              var rank = await handleExpectedCompletion(
                  listener: listener, metrics: metrics);
              if (bestRank < 0 || rank < bestRank) {
                bestRank = rank;
                bestName = metrics.name;
              }
              metrics.disable();
            }
            rankComparison.count(bestName);

            // If an overlay option is being used, remove the overlay applied
            // earlier
            if (options.overlay != CompletionMetricsOptions.OVERLAY_NONE) {
              _provider.removeOverlay(filePath);
            }
          }
        } catch (exception, stackTrace) {
          print('Exception caught analyzing: $filePath');
          print(exception.toString());
          print(stackTrace);
          resultCode = 1;
        }
      }
    }
  }

  List<protocol.CompletionSuggestion> _filterSuggestions(
      String prefix, List<protocol.CompletionSuggestion> suggestions) {
    // TODO(brianwilkerson) Replace this with a more realistic filtering
    //  algorithm.
    return suggestions
        .where((suggestion) => suggestion.completion.startsWith(prefix))
        .toList();
  }

  String _getOverlayContents(
      String contents, ExpectedCompletion expectedCompletion) {
    assert(contents.isNotEmpty);
    var offset = expectedCompletion.offset;
    var length = expectedCompletion.syntacticEntity.length;
    assert(offset >= 0);
    assert(length > 0);
    if (options.overlay == CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN) {
      return contents.substring(0, offset) +
          contents.substring(offset + length);
    } else if (options.overlay ==
        CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE) {
      return contents.substring(0, offset);
    } else {
      var removeToken = CompletionMetricsOptions.OVERLAY_REMOVE_TOKEN;
      var removeRest = CompletionMetricsOptions.OVERLAY_REMOVE_REST_OF_FILE;
      throw Exception('\'_getOverlayContents\' called with option other than'
          '$removeToken and $removeRest: ${options.overlay}');
    }
  }

  void _printSlowestResults(
      String title, List<CompletionResult> slowestResults) {
    printHeading(3, title);
    var needsBlankLine = false;
    for (var result in slowestResults) {
      var elapsedMS = result.elapsedMS;
      var expected = result.expectedCompletion;
      if (needsBlankLine) {
        print('');
      } else {
        needsBlankLine = true;
      }
      print('  Elapsed ms: $elapsedMS');
      print('  Completion: ${expected.completion}');
      print('  Completion kind: ${expected.kind}');
      print('  Element kind: ${expected.elementKind}');
      print('  Location: ${expected.location}');
    }
  }

  void _printWorstResults(String title, List<CompletionResult> worstResults) {
    List<String> suggestionRow(int rank, SuggestionData data) {
      var suggestion = data.suggestion;
      return [
        rank.toString(),
        suggestion.relevance.toString(),
        suggestion.completion,
        suggestion.kind.toString()
      ];
    }

    List<String> featuresRow(int rank, SuggestionData data) {
      var features = data.features;
      return [
        rank.toString(),
        for (var feature in features) feature.toStringAsFixed(4)
      ];
    }

    printHeading(3, title);
    var needsBlankLine = false;
    for (var result in worstResults) {
      var rank = result.place.rank;
      var actualSuggestion = result.actualSuggestion;
      var expected = result.expectedCompletion;

      var topSuggestions = result.topSuggestions;
      var topSuggestionCount = topSuggestions.length;

      var preceding = result.precedingRelevanceCounts;
      var precedingRelevances = preceding.keys.toList();
      precedingRelevances.sort();

      var suggestionsTable = [
        ['Rank', 'Relevance', 'Completion', 'Kind']
      ];
      for (var i = 0; i < topSuggestionCount; i++) {
        suggestionsTable.add(suggestionRow(i, topSuggestions[i]));
      }
      suggestionsTable.add(suggestionRow(rank, actualSuggestion));
      rightJustifyColumns(suggestionsTable, [0, 1]);

      var featuresTable = [
        [
          'Rank',
          'contextType',
          'elementKind',
          'hasDeprecated',
          'inheritanceDistance',
          'isConstant',
          'isNoSuchMethod',
          'keyword',
          'localVariableDistance',
          'startsWithDollar',
          'superMatches'
        ]
      ];
      for (var i = 0; i < topSuggestionCount; i++) {
        featuresTable.add(featuresRow(i, topSuggestions[i]));
      }
      featuresTable.add(featuresRow(rank, actualSuggestion));
      rightJustifyColumns(featuresTable, range(0, featuresTable[0].length));

      if (needsBlankLine) {
        print('');
      } else {
        needsBlankLine = true;
      }
      print('  Rank: $rank');
      print('  Location: ${expected.location}');
      print('  Comparison with the top $topSuggestionCount suggestions:');
      printTable(suggestionsTable);
      print('  Comparison of features with the top $topSuggestionCount '
          'suggestions:');
      printTable(featuresTable);
      print('  Preceding relevance scores and counts:');
      for (var relevance in precedingRelevances.reversed) {
        print('    $relevance: ${preceding[relevance]}');
      }
    }
  }

  /// Given some [ResolvedUnitResult] return the first error of high severity
  /// if such an error exists, `null` otherwise.
  static err.AnalysisError getFirstErrorOrNull(
      ResolvedUnitResult resolvedUnitResult) {
    for (var error in resolvedUnitResult.errors) {
      if (error.severity == Severity.error) {
        return error;
      }
    }
    return null;
  }

  static Place placementInSuggestionList(
      List<protocol.CompletionSuggestion> suggestions,
      ExpectedCompletion expectedCompletion) {
    var placeCounter = 1;
    for (var completionSuggestion in suggestions) {
      if (expectedCompletion.matches(completionSuggestion)) {
        return Place(placeCounter, suggestions.length);
      }
      placeCounter++;
    }
    return Place.none();
  }
}

/// The options specified on the command-line.
class CompletionMetricsOptions {
  /// A flag that causes the available suggestion sets to be used while
  /// computing suggestions.
  static const String AVAILABLE_SUGGESTIONS = 'available-suggestions';

  /// A flag that causes the output to be in markdown format.
  static const String MD = 'md';

  /// An option to control whether and how overlays should be produced.
  static const String OVERLAY = 'overlay';

  /// A mode indicating that no overlays should be produced.
  static const String OVERLAY_NONE = 'none';

  /// A mode indicating that everything from the completion offset to the end of
  /// the file should be removed.
  static const String OVERLAY_REMOVE_REST_OF_FILE = 'remove-rest-of-file';

  /// A mode indicating that the token whose offset is the same as the
  /// completion offset should be removed.
  static const String OVERLAY_REMOVE_TOKEN = 'remove-token';

  /// A flag that causes detailed information to be printed every time a
  /// completion request fails to produce a suggestions matching the expected
  /// suggestion.
  static const String PRINT_MISSED_COMPLETION_DETAILS =
      'print-missed-completion-details';

  /// A flag that causes summary information to be printed about the times that
  /// a completion request failed to produce a suggestions matching the expected
  /// suggestion.
  static const String PRINT_MISSED_COMPLETION_SUMMARY =
      'print-missed-completion-summary';

  /// A flag that causes information to be printed about places where no
  /// completion location was computed and about information that's missing in
  /// the completion tables.
  static const String PRINT_MISSING_INFORMATION = 'print-missing-information';

  /// A flag that causes information to be printed about the mrr score achieved
  /// at each completion location.
  static const String PRINT_MRR_BY_LOCATION = 'print-mrr-by-location';

  /// A flag that causes information to be printed about the completion requests
  /// that were the slowest to return suggestions.
  static const String PRINT_SLOWEST_RESULTS = 'print-slowest-results';

  /// A flag that causes information to be printed about the completion requests
  /// that had the worst mrr scores.
  static const String PRINT_WORST_RESULTS = 'print-worst-results';

  /// A flag indicating whether available suggestions should be enabled for this
  /// run.
  final bool availableSuggestions;

  /// A flag indicating whether the output should use markdown.
  final bool markdown;

  /// The overlay mode that should be used.
  final String overlay;

  /// A flag indicating whether information should be printed every time a
  /// completion request fails to produce a suggestions matching the expected
  /// suggestion.
  final bool printMissedCompletionDetails;

  /// A flag indicating whether information should be printed every time a
  /// completion request fails to produce a suggestions matching the expected
  /// suggestion.
  final bool printMissedCompletionSummary;

  /// A flag indicating whether information should be printed about places where
  /// no completion location was computed and about information that's missing
  /// in the completion tables.
  final bool printMissingInformation;

  /// A flag indicating whether information should be printed about the mrr
  /// score achieved at each completion location.
  final bool printMrrByLocation;

  /// A flag indicating whether information should be printed about the
  /// completion requests that were the slowest to return suggestions.
  final bool printSlowestResults;

  /// A flag indicating whether information should be printed about the
  /// completion requests that had the worst mrr scores.
  final bool printWorstResults;

  factory CompletionMetricsOptions(results) {
    return CompletionMetricsOptions._(
        availableSuggestions: results[AVAILABLE_SUGGESTIONS],
        markdown: results[MD],
        overlay: results[OVERLAY],
        printMissedCompletionDetails: results[PRINT_MISSED_COMPLETION_DETAILS],
        printMissedCompletionSummary: results[PRINT_MISSED_COMPLETION_SUMMARY],
        printMissingInformation: results[PRINT_MISSING_INFORMATION],
        printMrrByLocation: results[PRINT_MRR_BY_LOCATION],
        printSlowestResults: results[PRINT_SLOWEST_RESULTS],
        printWorstResults: results[PRINT_WORST_RESULTS]);
  }

  CompletionMetricsOptions._(
      {@required this.availableSuggestions,
      @required this.markdown,
      @required this.overlay,
      @required this.printMissedCompletionDetails,
      @required this.printMissedCompletionSummary,
      @required this.printMissingInformation,
      @required this.printMrrByLocation,
      @required this.printSlowestResults,
      @required this.printWorstResults})
      : assert(overlay == OVERLAY_NONE ||
            overlay == OVERLAY_REMOVE_TOKEN ||
            overlay == OVERLAY_REMOVE_REST_OF_FILE);
}

/// The result of a single completion.
class CompletionResult {
  final Place place;

  final CompletionRequestImpl request;

  final SuggestionData actualSuggestion;

  final List<SuggestionData> topSuggestions;

  final ExpectedCompletion expectedCompletion;

  final String completionLocation;

  final int elapsedMS;

  final Map<int, int> precedingRelevanceCounts;

  CompletionResult(
      this.place,
      this.request,
      this.actualSuggestion,
      this.topSuggestions,
      this.precedingRelevanceCounts,
      this.expectedCompletion,
      this.completionLocation,
      this.elapsedMS);

  /// Return an instance extracted from the decoded JSON [map].
  factory CompletionResult.fromJson(Map<String, dynamic> map) {
    var place = Place.fromJson(map['place'] as Map<String, dynamic>);
    var actualSuggestion = SuggestionData.fromJson(
        map['actualSuggestion'] as Map<String, dynamic>);
    var topSuggestions = (map['topSuggestions'] as List<dynamic>)
        ?.map((map) => SuggestionData.fromJson(map as Map<String, dynamic>))
        ?.toList();
    var precedingRelevanceCounts =
        (map['precedingRelevanceCounts'] as Map<String, dynamic>)
            ?.map((key, value) => MapEntry(int.parse(key), value as int));
    var expectedCompletion = ExpectedCompletion.fromJson(
        map['expectedCompletion'] as Map<String, dynamic>);
    var completionLocation = map['completionLocation'] as String;
    var elapsedMS = map['elapsedMS'] as int;
    return CompletionResult(
        place,
        null,
        actualSuggestion,
        topSuggestions,
        precedingRelevanceCounts,
        expectedCompletion,
        completionLocation,
        elapsedMS);
  }

  /// Return the completion group for the location at which completion was
  /// requested.
  CompletionGroup get group {
    var entity = expectedCompletion.syntacticEntity;
    var element = _getElement(entity);
    if (element != null) {
      var parent = element.enclosingElement;
      if (parent is ClassElement || parent is ExtensionElement) {
        if (_isStatic(element)) {
          return CompletionGroup.staticMember;
        } else {
          return CompletionGroup.instanceMember;
        }
      } else if (parent is CompilationUnitElement &&
          element is! ClassElement &&
          element is! ExtensionElement) {
        return CompletionGroup.topLevelMember;
      }
      if (element is ClassElement) {
        if (element.isEnum) {
          return CompletionGroup.enumElement;
        } else if (element.isMixin) {
          return CompletionGroup.mixinElement;
        }
        if (entity is SimpleIdentifier &&
            entity.parent is TypeName &&
            entity.parent.parent is ConstructorName &&
            entity.parent.parent.parent is InstanceCreationExpression) {
          return CompletionGroup.constructorElement;
        }
        return CompletionGroup.classElement;
      } else if (element is ExtensionElement) {
        return CompletionGroup.extensionElement;
      } else if (element is FunctionElement) {
        return CompletionGroup.localFunctionElement;
      } else if (element is LocalVariableElement) {
        return CompletionGroup.localVariableElement;
      } else if (element is ParameterElement) {
        return CompletionGroup.parameterElement;
      } else if (element is PrefixElement) {
        return CompletionGroup.prefixElement;
      } else if (element is TypeParameterElement) {
        return CompletionGroup.typeParameterElement;
      }
    }
    if (entity is SimpleIdentifier) {
      var name = entity.name;
      if (name == 'void') {
        return CompletionGroup.keywordVoid;
      } else if (name == 'dynamic') {
        return CompletionGroup.keywordDynamic;
      }
    }
    return CompletionGroup.unknown;
  }

  /// Return a map used to represent this completion result in a JSON structure.
  Map<String, dynamic> toJson() {
    return {
      'place': place.toJson(),
      'actualSuggestion': actualSuggestion.toJson(),
      if (topSuggestions != null)
        'topSuggestions':
            topSuggestions.map((suggestion) => suggestion.toJson()).toList(),
      if (precedingRelevanceCounts != null)
        'precedingRelevanceCounts': precedingRelevanceCounts
            .map((key, value) => MapEntry(key.toString(), value)),
      'expectedCompletion': expectedCompletion.toJson(),
      'completionLocation': completionLocation,
      'elapsedMS': elapsedMS,
    };
  }

  /// Return the element associated with the syntactic [entity], or `null` if
  /// there is no such element.
  Element _getElement(SyntacticEntity entity) {
    if (entity is SimpleIdentifier) {
      var element = entity.staticElement;
      if (element != null) {
        return element;
      }
      AstNode node = entity;
      while (node != null) {
        var parent = node.parent;
        if (parent is AssignmentExpression) {
          if (node == parent.leftHandSide) {
            return parent.readElement ?? parent.writeElement;
          }
          return null;
        } else if (parent is PrefixExpression) {
          if (parent.operator.type == TokenType.PLUS_PLUS ||
              parent.operator.type == TokenType.MINUS_MINUS) {
            return parent.readElement ?? parent.writeElement;
          }
        } else if (parent is PostfixExpression) {
          return parent.readElement ?? parent.writeElement;
        }
        node = parent;
      }
    }
    return null;
  }

  /// Return `true` if the [element] is static (either top-level or a static
  /// member of a class or extension).
  bool _isStatic(Element element) {
    if (element is ClassMemberElement) {
      return element.isStatic;
    } else if (element is ExecutableElement) {
      return element.isStatic;
    } else if (element is FieldElement) {
      return element.isStatic;
    } else if (element is VariableElement) {
      return element.isStatic;
    }
    return true;
  }
}

/// The data to be printed on a single line in the table of mrr values per
/// completion location.
class LocationTableLine {
  final String label;
  final double product;
  final int count;
  final double mrr;
  final double mrr_5;

  LocationTableLine(
      {@required this.label,
      @required this.product,
      @required this.count,
      @required this.mrr,
      @required this.mrr_5});
}

class MetricsSuggestionListener implements SuggestionListener {
  Map<protocol.CompletionSuggestion, List<double>> featureMap = {};

  List<double> cachedFeatures = const [
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0,
    0.0
  ];

  String missingCompletionLocation;

  String missingCompletionLocationTable;

  @override
  void builtSuggestion(protocol.CompletionSuggestion suggestion) {
    featureMap[suggestion] = cachedFeatures;
    cachedFeatures = const [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0];
  }

  @override
  void computedFeatures(
      {double contextType = 0.0,
      double elementKind = 0.0,
      double hasDeprecated = 0.0,
      double inheritanceDistance = 0.0,
      double isConstant = 0.0,
      double isNoSuchMethod = 0.0,
      double keyword = 0.0,
      double localVariableDistance = 0.0,
      double startsWithDollar = 0.0,
      double superMatches = 0.0}) {
    cachedFeatures = [
      contextType,
      elementKind,
      hasDeprecated,
      inheritanceDistance,
      isConstant,
      isNoSuchMethod,
      keyword,
      localVariableDistance,
      startsWithDollar,
      superMatches
    ];
  }

  @override
  void missingCompletionLocationAt(AstNode parent, SyntacticEntity child) {
    if (missingCompletionLocation == null) {
      String className(SyntacticEntity entity) {
        var className = entity.runtimeType.toString();
        if (className.endsWith('Impl')) {
          className = className.substring(0, className.length - 4);
        }
        return className;
      }

      var parentClass = className(parent);
      var childClass = className(child);
      missingCompletionLocation = '$parentClass/$childClass';
    }
  }

  @override
  void missingElementKindTableFor(String completionLocation) {
    missingCompletionLocationTable = completionLocation;
  }
}

/// A description of a pair of relevance tables to be used in an experiment.
class RelevanceTables {
  /// The name of the experiment using the tables.
  final String name;

  /// The relevance table used for element kinds.
  final Map<String, Map<protocol.ElementKind, ProbabilityRange>>
      elementKindRelevance;

  /// The relevance table used for keywords.
  final Map<String, Map<String, ProbabilityRange>> keywordRelevance;

  /// Initialize a newly created description of a pair of relevance tables.
  RelevanceTables(this.name, this.elementKindRelevance, this.keywordRelevance);
}

/// The information being remembered about an individual suggestion.
class SuggestionData {
  /// The suggestion that was produced.
  protocol.CompletionSuggestion suggestion;

  /// The values of the features used to compute the suggestion.
  List<double> features;

  SuggestionData(this.suggestion, this.features);

  /// Return an instance extracted from the decoded JSON [map].
  factory SuggestionData.fromJson(Map<String, dynamic> map) {
    return SuggestionData(
        protocol.CompletionSuggestion.fromJson(ResponseDecoder(null), '',
            map['suggestion'] as Map<String, dynamic>),
        (map['features'] as List<dynamic>).cast<double>());
  }

  /// Return a map used to represent this suggestion data in a JSON structure.
  Map<String, dynamic> toJson() {
    return {
      'suggestion': suggestion.toJson(),
      'features': features,
    };
  }
}

extension on CompletionGroup {
  String get name {
    switch (this) {
      case CompletionGroup.classElement:
        return 'class';
      case CompletionGroup.constructorElement:
        return 'constructor';
      case CompletionGroup.enumElement:
        return 'enum';
      case CompletionGroup.extensionElement:
        return 'extension';
      case CompletionGroup.instanceMember:
        return 'instance member';
      case CompletionGroup.labelElement:
        return 'label';
      case CompletionGroup.localFunctionElement:
        return 'local function';
      case CompletionGroup.localVariableElement:
        return 'local variable';
      case CompletionGroup.mixinElement:
        return 'mixin';
      case CompletionGroup.parameterElement:
        return 'parameter';
      case CompletionGroup.prefixElement:
        return 'prefix';
      case CompletionGroup.staticMember:
        return 'static member';
      case CompletionGroup.topLevelMember:
        return 'top level member';
      case CompletionGroup.typeParameterElement:
        return 'type parameter';

      case CompletionGroup.keywordDynamic:
        return 'keyword dynamic';
      case CompletionGroup.keywordVoid:
        return 'keyword void';

      case CompletionGroup.unknown:
        return 'unknown';
    }
    return '<unknown>';
  }
}

extension AvailableSuggestionsExtension on protocol.AvailableSuggestion {
  // TODO(jwren) I am not sure if we want CompletionSuggestionKind.INVOCATION in
  // call cases here, to iterate I need to figure out why this algorithm is
  // taking so much time.
  protocol.CompletionSuggestion toCompletionSuggestion(int relevance) =>
      protocol.CompletionSuggestion(
          protocol.CompletionSuggestionKind.INVOCATION,
          relevance,
          label,
          label.length,
          0,
          element.isDeprecated,
          false);
}