blob: 3816e5f863fe9a6e65188ac72531e5fed27a43df [file] [log] [blame]
// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:convert';
import 'dart:io';
import 'dart:math' as math;
import 'package:analyzer/file_system/file_system.dart';
import 'package:analyzer/file_system/physical_file_system.dart';
import 'package:args/command_runner.dart';
import 'package:intl/intl.dart';
import 'package:path/path.dart' as path;
import 'perf/benchmarks_impl.dart';
import 'perf/flutter_analyze_benchmark.dart';
Future main(List<String> args) async {
var benchmarks = <Benchmark>[
ColdAnalysisBenchmark(ServerBenchmark.das),
ColdAnalysisBenchmark(ServerBenchmark.lsp),
AnalysisBenchmark(ServerBenchmark.das),
AnalysisBenchmark(ServerBenchmark.lsp),
FlutterAnalyzeBenchmark(),
];
var runner =
CommandRunner('benchmark', 'A benchmark runner for the analysis server.');
runner.addCommand(ListCommand(benchmarks));
runner.addCommand(RunCommand(benchmarks));
runner.run(args);
}
String get analysisServerSrcPath {
var script = Platform.script.toFilePath(windows: Platform.isWindows);
var pkgPath = path.normalize(path.join(path.dirname(script), '..', '..'));
return path.join(pkgPath, 'analysis_server');
}
void deleteServerCache() {
// ~/.dartServer/.analysis-driver/
ResourceProvider resourceProvider = PhysicalResourceProvider.INSTANCE;
var stateLocation = resourceProvider.getStateLocation('.analysis-driver');
try {
if (stateLocation != null && stateLocation.exists) {
stateLocation.delete();
}
} catch (e) {
// ignore any exception
}
}
List<String> getProjectRoots({bool quick = false}) {
var script = Platform.script.toFilePath(windows: Platform.isWindows);
var pkgPath = path.normalize(path.join(path.dirname(script), '..', '..'));
return <String>[path.join(pkgPath, quick ? 'meta' : 'analysis_server')];
}
abstract class Benchmark {
final String id;
final String description;
final bool enabled;
/// One of 'memory', 'cpu', or 'group'.
final String kind;
Benchmark(this.id, this.description,
{this.enabled = true, this.kind = 'cpu'});
int get maxIterations => 0;
bool get needsSetup => false;
Future oneTimeCleanup() => Future.value();
Future oneTimeSetup() => Future.value();
Future<BenchMarkResult> run({
bool quick = false,
bool verbose = false,
});
Map toJson() =>
{'id': id, 'description': description, 'enabled': enabled, 'kind': kind};
@override
String toString() => '$id: $description';
}
class BenchMarkResult {
static final NumberFormat nf = NumberFormat.decimalPattern();
/// One of 'bytes', 'micros', or 'compound'.
final String kindName;
final int value;
BenchMarkResult(this.kindName, this.value);
BenchMarkResult combine(BenchMarkResult other) {
return BenchMarkResult(kindName, math.min(value, other.value));
}
Map toJson() => {kindName: value};
@override
String toString() => '$kindName: ${nf.format(value)}';
}
class CompoundBenchMarkResult extends BenchMarkResult {
final String name;
Map<String, BenchMarkResult> results = {};
CompoundBenchMarkResult(this.name) : super('compound', 0);
void add(String name, BenchMarkResult result) {
results[name] = result;
}
@override
BenchMarkResult combine(BenchMarkResult other) {
BenchMarkResult _combine(BenchMarkResult? a, BenchMarkResult? b) {
if (a == null) return b!;
if (b == null) return a;
return a.combine(b);
}
var o = other as CompoundBenchMarkResult;
var combined = CompoundBenchMarkResult(name);
var keys =
(<String>{}..addAll(results.keys)..addAll(o.results.keys)).toList();
for (var key in keys) {
combined.add(key, _combine(results[key], o.results[key]));
}
return combined;
}
@override
Map toJson() {
var m = <String, dynamic>{};
for (var entry in results.entries) {
m['$name-${entry.key}'] = entry.value.toJson();
}
return m;
}
@override
String toString() => '${toJson()}';
}
class ListCommand extends Command {
final List<Benchmark> benchmarks;
ListCommand(this.benchmarks) {
argParser.addFlag('machine',
negatable: false, help: 'Emit the list of benchmarks as json.');
}
@override
String get description => 'List available benchmarks.';
@override
String get invocation => '${runner!.executableName} $name';
@override
String get name => 'list';
@override
void run() {
if (argResults!['machine'] as bool) {
var map = <String, dynamic>{
'benchmarks': benchmarks.map((b) => b.toJson()).toList()
};
print(JsonEncoder.withIndent(' ').convert(map));
} else {
for (var benchmark in benchmarks) {
print('${benchmark.id}: ${benchmark.description}');
}
}
}
}
class RunCommand extends Command {
final List<Benchmark> benchmarks;
RunCommand(this.benchmarks) {
argParser.addFlag('quick',
negatable: false,
help: 'Run a quick version of the benchmark. This is not useful for '
'gathering accurate times,\nbut can be used to validate that the '
'benchmark works.');
argParser.addOption('repeat',
defaultsTo: '4', help: 'The number of times to repeat the benchmark.');
argParser.addFlag('verbose',
negatable: false,
help: 'Print all communication to and from the analysis server.');
}
@override
String get description => 'Run a given benchmark.';
@override
String get invocation => '${runner!.executableName} $name <benchmark-id>';
@override
String get name => 'run';
@override
Future run() async {
if (argResults!.rest.isEmpty) {
printUsage();
exit(1);
}
var benchmarkId = argResults!.rest.first;
var repeatCount = int.parse(argResults!['repeat'] as String);
var quick = argResults!['quick'];
var verbose = argResults!['verbose'];
var benchmark =
benchmarks.firstWhere((b) => b.id == benchmarkId, orElse: () {
print("Benchmark '$benchmarkId' not found.");
exit(1);
});
var actualIterations = repeatCount;
if (benchmark.maxIterations > 0) {
actualIterations = math.min(benchmark.maxIterations, repeatCount);
}
if (benchmark.needsSetup) {
print('Setting up $benchmarkId...');
await benchmark.oneTimeSetup();
}
try {
BenchMarkResult? result;
var time = Stopwatch()..start();
print('Running $benchmarkId $actualIterations times...');
for (var iteration = 0; iteration < actualIterations; iteration++) {
var newResult = await benchmark.run(
quick: quick,
verbose: verbose,
);
print(' $newResult');
result = result == null ? newResult : result.combine(newResult);
}
time.stop();
print('Finished in ${time.elapsed.inSeconds} seconds.\n');
var m = <String, dynamic>{
'benchmark': benchmarkId,
'result': result!.toJson()
};
print(json.encode(m));
await benchmark.oneTimeCleanup();
} catch (error, st) {
print('$benchmarkId threw exception: $error');
print(st);
exit(1);
}
}
}