Fix a bug in the benchmark script and enhance CSV output (#9955)
* Fix issue causing wasm to always be used even when useWasm = false
* Improve data quality of CSV output.
* pr comments
diff --git a/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart b/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart
index d0fb75c..8a8ba98 100644
--- a/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart
+++ b/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart
@@ -96,7 +96,7 @@
void writeHeaders({required int averageOf}) {
writeLines([
- 'Flutter DevTools performance benchmarks diff: dart2wasm diffed against dart2js.',
+ 'Flutter DevTools performance benchmarks diff: dart2js (baseline) vs dart2wasm (test).',
'Benchmark results were averaged over $averageOf benchmark run(s).',
'',
'These results were auto-generated by a script:',
@@ -105,9 +105,8 @@
]);
// Write the Flutter and DevTools commit hash for the benchmark run.
- // TODO(kenz): automatically detect these and write them to the CSV.
- const flutter = '<enter manually by running \'flutter --version\'>';
- const devtools = '<enter manually by running \'git log\'>';
+ final flutter = _detectFlutterVersion();
+ final devtools = _detectDevToolsCommit();
writeLines([
'Version info:',
'Flutter: $flutter',
@@ -120,7 +119,8 @@
writeLine([
'Benchmark Name',
'Metric',
- 'Value (micros)',
+ 'Baseline (micros)',
+ 'Test (micros)',
'Delta (micros)',
'Delta (%)',
]);
@@ -178,6 +178,49 @@
'(Google Sheets, Excel, etc.) for viewing.',
);
}
+
+ String _detectFlutterVersion() {
+ try {
+ final result = Process.runSync('flutter', [
+ '--version',
+ '--machine',
+ ], runInShell: true);
+ if (result.exitCode == 0) {
+ final json =
+ jsonDecode(result.stdout.toString()) as Map<String, Object?>;
+ final flutterVersion = json['flutterVersion'];
+ final frameworkRevision = json['frameworkRevision'];
+ if (flutterVersion != null && frameworkRevision != null) {
+ return '$flutterVersion (revision $frameworkRevision)';
+ }
+ return result.stdout.toString().trim();
+ }
+ } catch (_) {}
+
+ try {
+ final result = Process.runSync('flutter', [
+ '--version',
+ ], runInShell: true);
+ if (result.exitCode == 0) {
+ return result.stdout.toString().trim().split('\n').first;
+ }
+ } catch (_) {}
+
+ return '<unknown>';
+ }
+
+ String _detectDevToolsCommit() {
+ try {
+ final result = Process.runSync('git', [
+ 'rev-parse',
+ 'HEAD',
+ ], runInShell: true);
+ if (result.exitCode == 0) {
+ return result.stdout.toString().trim();
+ }
+ } catch (_) {}
+ return '<unknown>';
+ }
}
Future<BenchmarkResults> runBenchmarkOrUseExisting(
diff --git a/packages/devtools_app/benchmark/scripts/utils.dart b/packages/devtools_app/benchmark/scripts/utils.dart
index 6d7f24e..cb557c5 100644
--- a/packages/devtools_app/benchmark/scripts/utils.dart
+++ b/packages/devtools_app/benchmark/scripts/utils.dart
@@ -45,14 +45,22 @@
extension BenchmarkScoreExtension on BenchmarkScore {
List<String> toCsvLine() {
+ final deltaValue = delta;
+ final baselineValue = deltaValue != null ? value - deltaValue : null;
+ final String deltaPercent;
+ if (baselineValue == null) {
+ deltaPercent = '';
+ } else if (baselineValue == 0) {
+ deltaPercent = 'N/A';
+ } else {
+ deltaPercent = (deltaValue! / baselineValue).toString();
+ }
return [
metric, // Metric name
- value.toString(), // Value
- delta?.toString() ?? '', // Delta value
- // value - delta represents the baseline score.
- delta != null
- ? (delta! / (value - delta!)).toString()
- : '', // Delta % value
+ baselineValue?.toString() ?? '', // Baseline value
+ value.toString(), // Test value
+ deltaValue?.toString() ?? '', // Delta value
+ deltaPercent, // Delta % value
];
}
}
diff --git a/packages/devtools_app/benchmark/test_infra/common.dart b/packages/devtools_app/benchmark/test_infra/common.dart
index 32ab643..ec978f0 100644
--- a/packages/devtools_app/benchmark/test_infra/common.dart
+++ b/packages/devtools_app/benchmark/test_infra/common.dart
@@ -10,11 +10,9 @@
/// found" in DevTools.
const _benchmarkInitialPage = '';
-const _wasmQueryParameters = {'compiler': 'wasm'};
-
String benchmarkPath({required bool useWasm}) => Uri(
path: _benchmarkInitialPage,
- queryParameters: useWasm ? _wasmQueryParameters : null,
+ queryParameters: {'compiler': useWasm ? 'wasm' : 'js'},
).toString();
String generateBenchmarkEntryPoint({required bool useWasm}) {