[benchmark_harness]: add a bench command (#2091)
diff --git a/.github/workflows/benchmark_harness.yaml b/.github/workflows/benchmark_harness.yaml
index 5e783a4..4b83952 100644
--- a/.github/workflows/benchmark_harness.yaml
+++ b/.github/workflows/benchmark_harness.yaml
@@ -61,6 +61,11 @@
- uses: dart-lang/setup-dart@e51d8e571e22473a2ddebf0ef8a2123f0ab2c02c
with:
sdk: ${{ matrix.sdk }}
+ # Node 22 has wasmGC enabled, which allows the wasm tests to run!
+ - name: Setup Node.js 22
+ uses: actions/setup-node@v3
+ with:
+ node-version: 22
- id: install
name: Install dependencies
run: dart pub get
diff --git a/pkgs/benchmark_harness/CHANGELOG.md b/pkgs/benchmark_harness/CHANGELOG.md
index f0e24bf..87864a0 100644
--- a/pkgs/benchmark_harness/CHANGELOG.md
+++ b/pkgs/benchmark_harness/CHANGELOG.md
@@ -1,4 +1,6 @@
-## 2.3.2-wip
+## 2.4.0-wip
+
+- Added a `bench` command.
## 2.3.1
diff --git a/pkgs/benchmark_harness/README.md b/pkgs/benchmark_harness/README.md
index f156f23..d477808 100644
--- a/pkgs/benchmark_harness/README.md
+++ b/pkgs/benchmark_harness/README.md
@@ -114,3 +114,56 @@
This is the average amount of time it takes to run `run()` 10 times for
`BenchmarkBase` and once for `AsyncBenchmarkBase`.
> µs is an abbreviation for microseconds.
+
+## `bench` command
+
+A convenience command available in `package:benchmark_harness`.
+
+If a package depends on `benchmark_harness`, invoke the command by running
+
+```shell
+dart run benchmark_harness:bench
+```
+
+If not, you can use this command by activating it.
+
+```shell
+dart pub global activate benchmark_harness
+dart pub global run benchmark_harness:bench
+```
+
+Output from `dart run benchmark_harness:bench --help`
+
+```
+Runs a dart script in a number of runtimes.
+
+Meant to make it easy to run a benchmark executable across runtimes to validate
+performance impacts.
+
+-f, --flavor
+ [aot] Compile and run as a native binary.
+ [jit] Run as-is without compilation, using the just-in-time (JIT) runtime.
+ [js] Compile to JavaScript and run on node.
+ [wasm] Compile to WebAssembly and run on node.
+
+ --target The target script to compile and run.
+ (defaults to "benchmark/benchmark.dart")
+-h, --help Print usage information and quit.
+-v, --verbose Print the full stack trace if an exception is thrown.
+```
+
+Example usage:
+
+```shell
+dart run benchmark_harness:bench --flavor aot --target example/template.dart
+
+AOT - COMPILE
+/dart_installation/dart-sdk/bin/dart compile exe example/template.dart -o /temp_dir/bench_1747680526905_GtfAeM/out.exe
+
+Generated: /temp_dir/bench_1747680526905_GtfAeM/out.exe
+
+AOT - RUN
+/temp_dir/bench_1747680526905_GtfAeM/out.exe
+
+Template(RunTime): 0.005620051244379949 us.
+```
\ No newline at end of file
diff --git a/pkgs/benchmark_harness/bin/bench.dart b/pkgs/benchmark_harness/bin/bench.dart
new file mode 100644
index 0000000..43e1a91
--- /dev/null
+++ b/pkgs/benchmark_harness/bin/bench.dart
@@ -0,0 +1,39 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:benchmark_harness/src/bench_command/bench_options.dart';
+import 'package:benchmark_harness/src/bench_command/compile_and_run.dart';
+
+Future<void> main(List<String> args) async {
+ BenchOptions? options;
+
+ try {
+ options = BenchOptions.fromArgs(args);
+ if (options.help) {
+ print('''
+\nRuns a dart script in a number of runtimes.
+
+Meant to make it easy to run a benchmark executable across runtimes to validate
+performance impacts.
+''');
+ print(BenchOptions.usage);
+ return;
+ }
+
+ await compileAndRun(options);
+ } on FormatException catch (e) {
+ print(e.message);
+ print(BenchOptions.usage);
+ exitCode = 64; // command line usage error
+ } on BenchException catch (e, stack) {
+ print(e.message);
+ if (options?.verbose ?? true) {
+ print(e);
+ print(stack);
+ }
+ exitCode = e.exitCode;
+ }
+}
diff --git a/pkgs/benchmark_harness/lib/src/bench_command/bench_options.dart b/pkgs/benchmark_harness/lib/src/bench_command/bench_options.dart
new file mode 100644
index 0000000..e1ffa1c
--- /dev/null
+++ b/pkgs/benchmark_harness/lib/src/bench_command/bench_options.dart
@@ -0,0 +1,82 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'package:args/args.dart';
+
+enum RuntimeFlavor {
+ aot(help: 'Compile and run as a native binary.'),
+ jit(
+ help: 'Run as-is without compilation, '
+ 'using the just-in-time (JIT) runtime.',
+ ),
+ js(help: 'Compile to JavaScript and run on node.'),
+ wasm(help: 'Compile to WebAssembly and run on node.');
+
+ const RuntimeFlavor({required this.help});
+
+ final String help;
+}
+
+class BenchOptions {
+ BenchOptions({
+ required this.flavor,
+ required this.target,
+ this.help = false,
+ this.verbose = false,
+ }) {
+ if (!help && flavor.isEmpty) {
+ // This is the wrong exception to use, except that it's caught in the
+ // program, so it makes implementation easy.
+ throw const FormatException('At least one `flavor` must be provided', 64);
+ }
+ }
+
+ factory BenchOptions.fromArgs(List<String> args) {
+ final result = _parserForBenchOptions.parse(args);
+
+ if (result.rest.isNotEmpty) {
+ throw FormatException('All arguments must be provided via `--` options. '
+ 'Not sure what to do with "${result.rest.join()}".');
+ }
+
+ return BenchOptions(
+ flavor:
+ result.multiOption('flavor').map(RuntimeFlavor.values.byName).toSet(),
+ target: result.option('target')!,
+ help: result.flag('help'),
+ verbose: result.flag('verbose'),
+ );
+ }
+
+ final String target;
+
+ final Set<RuntimeFlavor> flavor;
+
+ final bool help;
+
+ final bool verbose;
+
+ static String get usage => _parserForBenchOptions.usage;
+
+ static final _parserForBenchOptions = ArgParser()
+ ..addMultiOption('flavor',
+ abbr: 'f',
+ allowed: RuntimeFlavor.values.map((e) => e.name),
+ allowedHelp: {
+ for (final flavor in RuntimeFlavor.values) flavor.name: flavor.help
+ })
+ ..addOption('target',
+ defaultsTo: 'benchmark/benchmark.dart',
+ help: 'The target script to compile and run.')
+ ..addFlag('help',
+ defaultsTo: false,
+ negatable: false,
+ help: 'Print usage information and quit.',
+ abbr: 'h')
+ ..addFlag('verbose',
+ defaultsTo: false,
+ negatable: false,
+ help: 'Print the full stack trace if an exception is thrown.',
+ abbr: 'v');
+}
diff --git a/pkgs/benchmark_harness/lib/src/bench_command/compile_and_run.dart b/pkgs/benchmark_harness/lib/src/bench_command/compile_and_run.dart
new file mode 100644
index 0000000..bd8f292
--- /dev/null
+++ b/pkgs/benchmark_harness/lib/src/bench_command/compile_and_run.dart
@@ -0,0 +1,186 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'bench_options.dart';
+
+// TODO(kevmoo): allow the user to specify custom flags – for compile and/or run
+
+Future<void> compileAndRun(BenchOptions options) async {
+ if (!FileSystemEntity.isFileSync(options.target)) {
+ throw BenchException(
+ 'The target Dart program `${options.target}` does not exist',
+ 2, // standard bash code for file doesn't exist
+ );
+ }
+
+ for (var mode in options.flavor) {
+ await _Runner(flavor: mode, target: options.target).run();
+ }
+}
+
+class BenchException implements Exception {
+ const BenchException(this.message, this.exitCode) : assert(exitCode > 0);
+ final String message;
+ final int exitCode;
+
+ @override
+ String toString() => 'BenchException: $message ($exitCode)';
+}
+
+/// Base name for output files.
+const _outputFileRoot = 'out';
+
+/// Denote the "stage" of the compile/run step for logging.
+enum _Stage { compile, run }
+
+/// Base class for runtime-specific runners.
+abstract class _Runner {
+ _Runner._({required this.target, required this.flavor})
+ : assert(FileSystemEntity.isFileSync(target), '$target is not a file');
+
+ factory _Runner({required RuntimeFlavor flavor, required String target}) {
+ return (switch (flavor) {
+ RuntimeFlavor.jit => _JITRunner.new,
+ RuntimeFlavor.aot => _AOTRunner.new,
+ RuntimeFlavor.js => _JSRunner.new,
+ RuntimeFlavor.wasm => _WasmRunner.new,
+ })(target: target);
+ }
+
+ final String target;
+ final RuntimeFlavor flavor;
+ late Directory _tempDirectory;
+
+ /// Executes the compile and run cycle.
+ ///
+ /// Takes care of creating and deleting the corresponding temp directory.
+ Future<void> run() async {
+ _tempDirectory = Directory.systemTemp
+ .createTempSync('bench_${DateTime.now().millisecondsSinceEpoch}_');
+ try {
+ await _runImpl();
+ } finally {
+ _tempDirectory.deleteSync(recursive: true);
+ }
+ }
+
+ /// Overridden in implementations to handle the compile and run cycle.
+ Future<void> _runImpl();
+
+ /// Executes the specific [executable] with the provided [args].
+ ///
+ /// Also prints out a nice message before execution denoting the [flavor] and
+ /// the [stage].
+ Future<void> _runProc(
+ _Stage stage, String executable, List<String> args) async {
+ print('''
+\n${flavor.name.toUpperCase()} - ${stage.name.toUpperCase()}
+$executable ${args.join(' ')}
+''');
+
+ final proc = await Process.start(executable, args,
+ mode: ProcessStartMode.inheritStdio);
+
+ final exitCode = await proc.exitCode;
+
+ if (exitCode != 0) {
+ throw ProcessException(executable, args, 'Process errored', exitCode);
+ }
+ }
+
+ String _outputFile(String ext) =>
+ _tempDirectory.uri.resolve('$_outputFileRoot.$ext').toFilePath();
+}
+
+class _JITRunner extends _Runner {
+ _JITRunner({required super.target}) : super._(flavor: RuntimeFlavor.jit);
+
+ @override
+ Future<void> _runImpl() async {
+ await _runProc(_Stage.run, Platform.executable, [target]);
+ }
+}
+
+class _AOTRunner extends _Runner {
+ _AOTRunner({required super.target}) : super._(flavor: RuntimeFlavor.aot);
+
+ @override
+ Future<void> _runImpl() async {
+ final outFile = _outputFile('exe');
+ await _runProc(_Stage.compile, Platform.executable, [
+ 'compile',
+ 'exe',
+ target,
+ '-o',
+ outFile,
+ ]);
+
+ await _runProc(_Stage.run, outFile, []);
+ }
+}
+
+class _JSRunner extends _Runner {
+ _JSRunner({required super.target}) : super._(flavor: RuntimeFlavor.js);
+
+ @override
+ Future<void> _runImpl() async {
+ final outFile = _outputFile('js');
+ await _runProc(_Stage.compile, Platform.executable, [
+ 'compile',
+ 'js',
+ target,
+ '-O4', // default for Flutter
+ '-o',
+ outFile,
+ ]);
+
+ await _runProc(_Stage.run, 'node', [outFile]);
+ }
+}
+
+class _WasmRunner extends _Runner {
+ _WasmRunner({required super.target}) : super._(flavor: RuntimeFlavor.wasm);
+
+ @override
+ Future<void> _runImpl() async {
+ final outFile = _outputFile('wasm');
+ await _runProc(_Stage.compile, Platform.executable, [
+ 'compile',
+ 'wasm',
+ target,
+ '-O2', // default for Flutter
+ '-o',
+ outFile,
+ ]);
+
+ final jsFile =
+ File.fromUri(_tempDirectory.uri.resolve('$_outputFileRoot.js'));
+ jsFile.writeAsStringSync(_wasmInvokeScript);
+
+ await _runProc(_Stage.run, 'node', [jsFile.path]);
+ }
+
+ static const _wasmInvokeScript = '''
+import { readFile } from 'node:fs/promises'; // For async file reading
+import { fileURLToPath } from 'url';
+import { dirname, join } from 'path';
+
+// Get the current directory name
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+const wasmFilePath = join(__dirname, '$_outputFileRoot.wasm');
+const wasmBytes = await readFile(wasmFilePath);
+
+const mjsFilePath = join(__dirname, '$_outputFileRoot.mjs');
+const dartModule = await import(mjsFilePath);
+const {compile} = dartModule;
+
+const compiledApp = await compile(wasmBytes);
+const instantiatedApp = await compiledApp.instantiate({});
+await instantiatedApp.invokeMain();
+''';
+}
diff --git a/pkgs/benchmark_harness/pubspec.yaml b/pkgs/benchmark_harness/pubspec.yaml
index 8561c2a..b2ad9a0 100644
--- a/pkgs/benchmark_harness/pubspec.yaml
+++ b/pkgs/benchmark_harness/pubspec.yaml
@@ -1,5 +1,5 @@
name: benchmark_harness
-version: 2.3.2-wip
+version: 2.4.0-wip
description: The official Dart project benchmark harness.
repository: https://github.com/dart-lang/tools/tree/main/pkgs/benchmark_harness
issue_tracker: https://github.com/dart-lang/tools/issues?q=is%3Aissue+is%3Aopen+label%3Apackage%3Abenchmark_harness
@@ -10,9 +10,13 @@
environment:
sdk: ^3.2.0
+dependencies:
+ args: ^2.5.0
+
dev_dependencies:
- build_runner: ^2.0.0
- build_web_compilers: ^4.0.0
dart_flutter_team_lints: ^3.0.0
- path: ^1.8.0
- test: ^1.16.0
+ path: ^1.9.0
+ test: ^1.25.7
+
+executables:
+ bench:
diff --git a/pkgs/benchmark_harness/test/bench_command_test.dart b/pkgs/benchmark_harness/test/bench_command_test.dart
new file mode 100644
index 0000000..7760416
--- /dev/null
+++ b/pkgs/benchmark_harness/test/bench_command_test.dart
@@ -0,0 +1,127 @@
+// Copyright (c) 2025, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+@TestOn('vm')
+library;
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:benchmark_harness/src/bench_command/bench_options.dart';
+import 'package:benchmark_harness/src/bench_command/compile_and_run.dart';
+import 'package:test/test.dart';
+
+void main() {
+ test('readme', () async {
+ final output =
+ await Process.run(Platform.executable, ['bin/bench.dart', '--help']);
+
+ expect(output.exitCode, 0);
+ // Sadly, the help output likes to include trailing spaces that don't
+ // copy paste nicely or consistently into the README.
+ final trimmed = LineSplitter.split(output.stdout as String)
+ .map((e) => e.trimRight())
+ .join('\n');
+
+ final readmeFile = File('README.md').readAsStringSync();
+
+ expect(
+ readmeFile,
+ contains(trimmed),
+ );
+ });
+
+ group('invoke the command', () {
+ late final Directory tempDir;
+ late final String testFilePath;
+
+ setUpAll(() {
+ tempDir = Directory.systemTemp.createTempSync('benchtest_');
+ testFilePath = tempDir.uri.resolve('input.dart').toFilePath();
+ File(testFilePath)
+ ..create()
+ ..writeAsStringSync(_testDartFile);
+ });
+
+ tearDownAll(() {
+ tempDir.deleteSync(recursive: true);
+ });
+
+ group('BenchOptions.fromArgs', () {
+ test('options parsing', () async {
+ final options = BenchOptions.fromArgs(
+ ['--flavor', 'aot,jit', '--target', testFilePath],
+ );
+
+ await expectLater(
+ () => compileAndRun(options),
+ prints(
+ stringContainsInOrder([
+ 'AOT - COMPILE',
+ testFilePath,
+ 'AOT - RUN',
+ 'JIT - RUN',
+ testFilePath,
+ ]),
+ ),
+ );
+ });
+
+ test('rest args not supported', () async {
+ expect(
+ () => BenchOptions.fromArgs(
+ ['--flavor', 'aot,jit', testFilePath],
+ ),
+ throwsFormatException,
+ );
+ });
+ });
+
+ for (var bench in RuntimeFlavor.values) {
+ test('$bench', () async {
+ await expectLater(
+ () => compileAndRun(
+ BenchOptions(flavor: {bench}, target: testFilePath)),
+ prints(
+ stringContainsInOrder(
+ [
+ if (bench != RuntimeFlavor.jit) ...[
+ '${bench.name.toUpperCase()} - COMPILE',
+ testFilePath,
+ ],
+ '${bench.name.toUpperCase()} - RUN'
+ ],
+ ),
+ ),
+ );
+ }, skip: _skipWasm(bench));
+ }
+ });
+}
+
+// Can remove this once the min tested SDK on GitHub is >= 3.7
+String? _skipWasm(RuntimeFlavor flavor) {
+ if (flavor != RuntimeFlavor.wasm) {
+ return null;
+ }
+ final versionBits = Platform.version.split('.');
+ final versionValues = versionBits.take(2).map(int.parse).toList();
+
+ return switch ((versionValues[0], versionValues[1])) {
+ // If major is greater than 3, it's definitely >= 3.7
+ (int m, _) when m > 3 => null,
+ // If major is 3, check the minor version
+ (3, int n) when n >= 7 => null,
+ // All other cases (major < 3, or major is 3 but minor < 7)
+ _ => 'Requires Dart >= 3.7',
+ };
+}
+
+const _testDartFile = '''
+void main() {
+ // outputs 0 is JS
+ // 8589934592 everywhere else
+ print(1 << 33);
+}
+''';
