Fix harness to not call timer repeatedly in the measured loop. (#38)
Co-authored-by: Ömer Sinan Ağacan <omersa@google.com>
Co-authored-by: Vyacheslav Egorov <vegorov@google.com>
diff --git a/CHANGELOG.md b/CHANGELOG.md
index c6d581f..7c808cf 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,3 +1,9 @@
+## 2.2.0
+
+- Change measuring algorithm in `BenchmarkBase` to avoid calling stopwatch
+methods repeatedly in the measuring loop. This makes measurement work better
+for `run` methods which are small themselves.
+
## 2.1.0
- Add AsyncBenchmarkBase.
diff --git a/lib/benchmark_harness.dart b/lib/benchmark_harness.dart
index 2809777..6170115 100644
--- a/lib/benchmark_harness.dart
+++ b/lib/benchmark_harness.dart
@@ -5,6 +5,7 @@
library benchmark_harness;
import 'dart:async';
+import 'dart:math' as math;
part 'src/benchmark_base.dart';
part 'src/async_benchmark_base.dart';
diff --git a/lib/src/benchmark_base.dart b/lib/src/benchmark_base.dart
index b8a08a7..4a6acfa 100644
--- a/lib/src/benchmark_base.dart
+++ b/lib/src/benchmark_base.dart
@@ -4,63 +4,97 @@
part of benchmark_harness;
+const int _minimumMeasureDurationMillis = 2000;
+
class BenchmarkBase {
final String name;
final ScoreEmitter emitter;
- // Empty constructor.
const BenchmarkBase(this.name, {this.emitter = const PrintEmitter()});
- // The benchmark code.
- // This function is not used, if both [warmup] and [exercise] are overwritten.
+ /// The benchmark code.
+ ///
+ /// This function is not used, if both [warmup] and [exercise] are overwritten.
void run() {}
- // Runs a short version of the benchmark. By default invokes [run] once.
+ /// Runs a short version of the benchmark. By default invokes [run] once.
void warmup() {
run();
}
- // Exercises the benchmark. By default invokes [run] 10 times.
+ /// Exercises the benchmark. By default invokes [run] 10 times.
void exercise() {
for (var i = 0; i < 10; i++) {
run();
}
}
- // Not measured setup code executed prior to the benchmark runs.
+ /// Not measured setup code executed prior to the benchmark runs.
void setup() {}
- // Not measures teardown code executed after the benchmark runs.
+ /// Not measured teardown code executed after the benchmark runs.
void teardown() {}
- // Measures the score for this benchmark by executing it repeatedly until
- // time minimum has been reached.
- static double measureFor(void Function() f, int minimumMillis) {
- var minimumMicros = minimumMillis * 1000;
- var iter = 0;
- var watch = Stopwatch();
- watch.start();
- var elapsed = 0;
- while (elapsed < minimumMicros) {
- f();
- elapsed = watch.elapsedMicroseconds;
- iter++;
+ /// Measures the score for this benchmark by executing it enough times
+ /// to reach [minimumMillis].
+ static _Measurement _measureForImpl(void Function() f, int minimumMillis) {
+ final minimumMicros = minimumMillis * 1000;
+ var iter = 2;
+ final watch = Stopwatch()..start();
+ while (true) {
+ watch.reset();
+ for (var i = 0; i < iter; i++) {
+ f();
+ }
+ final elapsed = watch.elapsedMicroseconds;
+ final measurement = _Measurement(elapsed, iter);
+ if (measurement.elapsedMicros >= minimumMicros) {
+ return measurement;
+ }
+
+ iter = measurement.estimateIterationsNeededToReach(
+ minimumMicros: minimumMicros);
}
- return elapsed / iter;
}
- // Measures the score for the benchmark and returns it.
+ /// Measures the score for this benchmark by executing it repeatedly until
+ /// time minimum has been reached.
+ static double measureFor(void Function() f, int minimumMillis) =>
+ _measureForImpl(f, minimumMillis).score;
+
+ /// Measures the score for the benchmark and returns it.
double measure() {
setup();
// Warmup for at least 100ms. Discard result.
- measureFor(warmup, 100);
+ _measureForImpl(warmup, 100);
// Run the benchmark for at least 2000ms.
- var result = measureFor(exercise, 2000);
+ var result = _measureForImpl(exercise, _minimumMeasureDurationMillis);
teardown();
- return result;
+ return result.score;
}
void report() {
emitter.emit(name, measure());
}
}
+
+class _Measurement {
+ final int elapsedMicros;
+ final int iterations;
+
+ _Measurement(this.elapsedMicros, this.iterations);
+
+ double get score => elapsedMicros / iterations;
+
+ int estimateIterationsNeededToReach({required int minimumMicros}) {
+ final elapsed = roundDownToMillisecond(elapsedMicros);
+ return elapsed == 0
+ ? iterations * 1000
+ : (iterations * math.max(minimumMicros / elapsed, 1.5)).ceil();
+ }
+
+ static int roundDownToMillisecond(int micros) => (micros ~/ 1000) * 1000;
+
+ @override
+ String toString() => '$elapsedMicros in $iterations iterations';
+}
diff --git a/pubspec.yaml b/pubspec.yaml
index 4c436c6..62675b9 100644
--- a/pubspec.yaml
+++ b/pubspec.yaml
@@ -1,5 +1,5 @@
name: benchmark_harness
-version: 2.1.0
+version: 2.2.0
description: The official Dart project benchmark harness.
repository: https://github.com/dart-lang/benchmark_harness
