pkg/analysis_server/benchmark/benchmarks.dart

// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:convert';
import 'dart:io';
import 'dart:math' as math;

import 'package:analyzer/file_system/file_system.dart';
import 'package:analyzer/file_system/physical_file_system.dart';
import 'package:analyzer_utilities/package_root.dart';
import 'package:args/command_runner.dart';
import 'package:path/path.dart' as path;

import 'perf/benchmarks_impl.dart';
import 'perf/flutter_analyze_benchmark.dart';
import 'perf/flutter_completion_benchmark.dart';

Future main(List<String> args) async {
  var benchmarks = <Benchmark>[
    ColdAnalysisBenchmark(ServerBenchmark.das),
    ColdAnalysisBenchmark(ServerBenchmark.lsp),
    AnalysisBenchmark(ServerBenchmark.das),
    AnalysisBenchmark(ServerBenchmark.lsp),
    FlutterAnalyzeBenchmark(),
    FlutterCompletionBenchmark.das,
    FlutterCompletionBenchmark.lsp,
  ];

  var runner = CommandRunner(
    'benchmark',
    'A benchmark runner for the analysis server.',
  );
  runner.addCommand(ListCommand(benchmarks));
  runner.addCommand(RunCommand(benchmarks));
  runner.run(args);
}

String get analysisServerSrcPath {
  return path.join(packageRoot, 'analysis_server');
}

void deleteServerCache() {
  // ~/.dartServer/.analysis-driver/
  ResourceProvider resourceProvider = PhysicalResourceProvider.INSTANCE;
  var stateLocation = resourceProvider.getStateLocation('.analysis-driver');
  try {
    stateLocation?.delete();
  } catch (e) {
    // ignore any exception
  }
}

List<String> getProjectRoots({bool quick = false}) {
  return [path.join(packageRoot, quick ? 'meta' : 'analysis_server')];
}

abstract class Benchmark {
  final String id;
  final String description;
  final bool enabled;

  /// One of 'memory', 'cpu', or 'group'.
  final String kind;

  Benchmark(this.id, this.description,
      {this.enabled = true, required this.kind});

  int get maxIterations => 0;

  bool get needsSetup => false;

  Future oneTimeCleanup() => Future.value();

  Future oneTimeSetup() => Future.value();

  Future<BenchMarkResult> run({
    required String dartSdkPath,
    bool quick = false,
    bool verbose = false,
  });

  Map toJson() =>
      {'id': id, 'description': description, 'enabled': enabled, 'kind': kind};

  @override
  String toString() => '$id: $description';
}

class BenchMarkResult {
  /// One of 'bytes', 'micros', or 'compound'.
  final String kindName;

  final int value;

  BenchMarkResult(this.kindName, this.value);

  BenchMarkResult combine(BenchMarkResult other) {
    return BenchMarkResult(kindName, math.min(value, other.value));
  }

  Map toJson() => {kindName: value};

  @override
  String toString() => '$kindName: $value';
}

class CompoundBenchMarkResult extends BenchMarkResult {
  final String name;

  Map<String, BenchMarkResult> results = {};

  CompoundBenchMarkResult(this.name) : super('compound', 0);

  void add(String name, BenchMarkResult result) {
    results[name] = result;
  }

  @override
  BenchMarkResult combine(BenchMarkResult other) {
    BenchMarkResult combine(BenchMarkResult? a, BenchMarkResult? b) {
      if (a == null) return b!;
      if (b == null) return a;
      return a.combine(b);
    }

    var o = other as CompoundBenchMarkResult;

    var combined = CompoundBenchMarkResult(name);
    var keys = (<String>{}
          ..addAll(results.keys)
          ..addAll(o.results.keys))
        .toList();

    for (var key in keys) {
      combined.add(key, combine(results[key], o.results[key]));
    }

    return combined;
  }

  @override
  Map toJson() {
    var m = <String, dynamic>{};
    for (var entry in results.entries) {
      m['$name-${entry.key}'] = entry.value.toJson();
    }
    return m;
  }

  @override
  String toString() => '${toJson()}';
}

/// This interface is implemented by benchmarks that need to know the location
/// of the Flutter repository.
abstract class FlutterBenchmark {
  /// Must be called exactly one time.
  set flutterRepositoryPath(String path);
}

class ListCommand extends Command {
  final List<Benchmark> benchmarks;

  ListCommand(this.benchmarks) {
    argParser.addFlag('machine',
        negatable: false, help: 'Emit the list of benchmarks as json.');
  }

  @override
  String get description => 'List available benchmarks.';

  @override
  String get invocation => '${runner!.executableName} $name';

  @override
  String get name => 'list';

  @override
  void run() {
    if (argResults!['machine'] as bool) {
      var map = <String, dynamic>{
        'benchmarks': benchmarks.map((b) => b.toJson()).toList()
      };
      print(JsonEncoder.withIndent('  ').convert(map));
    } else {
      for (var benchmark in benchmarks) {
        print('${benchmark.id}: ${benchmark.description}');
      }
    }
  }
}

class RunCommand extends Command {
  final List<Benchmark> benchmarks;

  RunCommand(this.benchmarks) {
    argParser.addOption('dart-sdk',
        help: 'The absolute normalized path of the Dart SDK.');
    argParser.addOption('flutter-repository',
        help: 'The absolute normalized path of the Flutter repository.');
    argParser.addFlag('quick',
        negatable: false,
        help: 'Run a quick version of the benchmark. This is not useful for '
            'gathering accurate times,\nbut can be used to validate that the '
            'benchmark works.');
    argParser.addOption('repeat',
        defaultsTo: '4', help: 'The number of times to repeat the benchmark.');
    argParser.addFlag('verbose',
        negatable: false,
        help: 'Print all communication to and from the analysis server.');
  }

  @override
  String get description => 'Run a given benchmark.';

  @override
  String get invocation => '${runner!.executableName} $name <benchmark-id>';

  @override
  String get name => 'run';

  @override
  Future run() async {
    var args = argResults;
    if (args == null) {
      throw StateError('argResults have not been set');
    }
    if (args.rest.isEmpty) {
      printUsage();
      exit(1);
    }

    var benchmarkId = args.rest.first;
    var repeatCount = int.parse(args['repeat'] as String);
    var dartSdkPath = args['dart-sdk'] as String?;
    var flutterRepository = args['flutter-repository'] as String?;
    var quick = args['quick'] as bool;
    var verbose = args['verbose'] as bool;

    var benchmark =
        benchmarks.firstWhere((b) => b.id == benchmarkId, orElse: () {
      print("Benchmark '$benchmarkId' not found.");
      exit(1);
    });

    dartSdkPath ??= path.dirname(path.dirname(Platform.resolvedExecutable));

    if (benchmark is FlutterBenchmark) {
      if (flutterRepository != null) {
        if (path.isAbsolute(flutterRepository) &&
            path.normalize(flutterRepository) == flutterRepository) {
          (benchmark as FlutterBenchmark).flutterRepositoryPath =
              flutterRepository;
        } else {
          print('The path must be absolute and normalized: $flutterRepository');
          exit(1);
        }
      } else {
        print('The option --flutter-repository is required to '
            "run '$benchmarkId'.");
        exit(1);
      }
    }

    var actualIterations = repeatCount;
    if (benchmark.maxIterations > 0) {
      actualIterations = math.min(benchmark.maxIterations, repeatCount);
    }

    if (benchmark.needsSetup) {
      print('Setting up $benchmarkId...');
      await benchmark.oneTimeSetup();
    }

    try {
      BenchMarkResult? result;
      var time = Stopwatch()..start();
      print('Running $benchmarkId $actualIterations times...');

      for (var iteration = 0; iteration < actualIterations; iteration++) {
        var newResult = await benchmark.run(
          dartSdkPath: dartSdkPath,
          quick: quick,
          verbose: verbose,
        );
        print('  $newResult');
        result = result == null ? newResult : result.combine(newResult);
      }

      time.stop();
      print('Finished in ${time.elapsed.inSeconds} seconds.\n');
      var m = <String, dynamic>{
        'benchmark': benchmarkId,
        'result': result!.toJson()
      };
      print(json.encode(m));

      await benchmark.oneTimeCleanup();
    } catch (error, st) {
      print('$benchmarkId threw exception: $error');
      print(st);
      exit(1);
    }
  }
}