pkg/analysis_server/benchmark/perf/benchmarks_impl.dart

// Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:io';

import 'package:path/path.dart' as path;

import '../benchmarks.dart';
import 'memory_tests.dart';

/// benchmarks:
///   - analysis-server-warm-analysis
///   - analysis-server-warm-memory
///   - analysis-server-edit
///   - analysis-server-completion
class AnalysisBenchmark extends Benchmark {
  final AbstractBenchmarkTest Function() testConstructor;

  new(ServerBenchmark benchmarkTest)
    : testConstructor = benchmarkTest.testConstructor,
      super(
        benchmarkTest.id,
        '${benchmarkTest.name} benchmarks of a large project, with an existing '
        'driver cache.',
        kind: 'group',
      );

  @override
  Future<BenchMarkResult> run({
    required String dartSdkPath,
    bool quick = false,
    bool verbose = false,
  }) async {
    var stopwatch = Stopwatch()..start();

    var test = testConstructor();
    if (verbose) {
      test.debugStdio();
    }
    await test.setUp(dartSdkPath, getProjectRoots(quick: quick));
    await test.analysisFinished;

    stopwatch.stop();
    var usedBytes = await test.getMemoryUsage();

    var result = CompoundBenchMarkResult(id);
    result.add(
      'warm-analysis',
      BenchMarkResult('micros', stopwatch.elapsedMicroseconds),
    );
    result.add('warm-memory', BenchMarkResult('bytes', usedBytes));

    if (!quick) {
      // change timing
      var editMicros = await _calcEditTiming(test);
      result.add('edit', BenchMarkResult('micros', editMicros));

      // code completion
      var completionMicros = await _calcCompletionTiming(test);
      result.add('completion', BenchMarkResult('micros', completionMicros));
    }

    await test.shutdown();

    return result;
  }

  Future<int> _calcCompletionTiming(AbstractBenchmarkTest test) async {
    const kGroupCount = 10;

    var filePath = path.join(
      analysisServerSrcPath,
      'lib',
      'src',
      'analysis_server.dart',
    );
    var contents = File(filePath).readAsStringSync();

    await test.openFile(filePath, contents);

    var completionCount = 0;
    var stopwatch = Stopwatch()..start();

    Future<void> complete(int offset) async {
      await test.complete(filePath, offset, isWarmUp: false);
      completionCount++;
    }

    for (var i = 0; i < kGroupCount; i++) {
      var startIndex = i * (contents.length ~/ (kGroupCount + 2));
      // Look for a line with a period in it that ends with a semi-colon.
      var index = contents.indexOf(
        RegExp(r'\..*;$', multiLine: true),
        startIndex,
      );

      await complete(index - 10);
      await complete(index - 1);
      await complete(index);
      await complete(index + 1);
      await complete(index + 10);

      if (i + 1 < kGroupCount) {
        // mutate
        index = contents.indexOf(';', index);
        contents =
            '${contents.substring(0, index + 1)} ${contents.substring(index + 1)}';
        await test.updateFile(filePath, contents);
      }
    }

    stopwatch.stop();

    await test.closeFile(filePath);

    return stopwatch.elapsedMicroseconds ~/ completionCount;
  }

  Future<int> _calcEditTiming(AbstractBenchmarkTest test) async {
    const kGroupCount = 5;

    var filePath = path.join(
      analysisServerSrcPath,
      'lib',
      'src',
      'analysis_server.dart',
    );
    var contents = File(filePath).readAsStringSync();

    // Get the future for `analysisFinished` *before* the call to [openFile]
    // as the analysis triggered by opening is sometimes so fast that it
    // finishes before starting to listen if only starting *after* the call to
    // [openFile].
    Future<void> analysisFinished = test.analysisFinished;
    await test.openFile(filePath, contents);
    await analysisFinished;

    var stopwatch = Stopwatch()..start();

    for (var i = 0; i < kGroupCount; i++) {
      var startIndex = i * (contents.length ~/ (kGroupCount + 2));
      var index = contents.indexOf(';', startIndex);
      contents =
          '${contents.substring(0, index + 1)} ${contents.substring(index + 1)}';
      await test.updateFile(filePath, contents);
      await test.analysisFinished;
    }

    stopwatch.stop();

    await test.closeFile(filePath);

    return stopwatch.elapsedMicroseconds ~/ kGroupCount;
  }
}

/// benchmarks:
///   - analysis-server-cold-analysis
///   - analysis-server-cold-memory
class ColdAnalysisBenchmark extends Benchmark {
  final AbstractBenchmarkTest Function() testConstructor;

  new(ServerBenchmark benchmarkTest)
    : testConstructor = benchmarkTest.testConstructor,
      super(
        '${benchmarkTest.id}-cold',
        '${benchmarkTest.name} benchmarks of a large project on start-up, no '
            'existing driver cache.',
        kind: 'group',
      );

  @override
  int get maxIterations => 3;

  @override
  Future<BenchMarkResult> run({
    required String dartSdkPath,
    bool quick = false,
    bool verbose = false,
  }) async {
    if (!quick) {
      deleteServerCache();
    }

    var stopwatch = Stopwatch()..start();

    var test = testConstructor();
    await test.setUp(dartSdkPath, getProjectRoots(quick: quick));
    await test.analysisFinished;

    stopwatch.stop();
    var usedBytes = await test.getMemoryUsage();

    var result = CompoundBenchMarkResult(id);
    result.add(
      'analysis',
      BenchMarkResult('micros', stopwatch.elapsedMicroseconds),
    );
    result.add('memory', BenchMarkResult('bytes', usedBytes));

    await test.shutdown();

    return result;
  }
}

class ServerBenchmark {
  static final das = ServerBenchmark(
    'analysis-server',
    'Analysis Server',
    AnalysisServerBenchmarkTest.new,
  );
  static final lsp = ServerBenchmark(
    'lsp-analysis-server',
    'LSP Analysis Server',
    LspAnalysisServerBenchmarkTest.new,
  );
  final String id;

  final String name;
  final AbstractBenchmarkTest Function() testConstructor;

  new(this.id, this.name, this.testConstructor);
}