Fix a bug in the benchmark script and enhance CSV output (#9955) * Fix issue causing wasm to always be used even when useWasm = false * Improve data quality of CSV output. * pr comments
diff --git a/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart b/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart index d0fb75c..8a8ba98 100644 --- a/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart +++ b/packages/devtools_app/benchmark/scripts/dart2wasm_performance_diff.dart
@@ -96,7 +96,7 @@ void writeHeaders({required int averageOf}) { writeLines([ - 'Flutter DevTools performance benchmarks diff: dart2wasm diffed against dart2js.', + 'Flutter DevTools performance benchmarks diff: dart2js (baseline) vs dart2wasm (test).', 'Benchmark results were averaged over $averageOf benchmark run(s).', '', 'These results were auto-generated by a script:', @@ -105,9 +105,8 @@ ]); // Write the Flutter and DevTools commit hash for the benchmark run. - // TODO(kenz): automatically detect these and write them to the CSV. - const flutter = '<enter manually by running \'flutter --version\'>'; - const devtools = '<enter manually by running \'git log\'>'; + final flutter = _detectFlutterVersion(); + final devtools = _detectDevToolsCommit(); writeLines([ 'Version info:', 'Flutter: $flutter', @@ -120,7 +119,8 @@ writeLine([ 'Benchmark Name', 'Metric', - 'Value (micros)', + 'Baseline (micros)', + 'Test (micros)', 'Delta (micros)', 'Delta (%)', ]); @@ -178,6 +178,49 @@ '(Google Sheets, Excel, etc.) for viewing.', ); } + + String _detectFlutterVersion() { + try { + final result = Process.runSync('flutter', [ + '--version', + '--machine', + ], runInShell: true); + if (result.exitCode == 0) { + final json = + jsonDecode(result.stdout.toString()) as Map<String, Object?>; + final flutterVersion = json['flutterVersion']; + final frameworkRevision = json['frameworkRevision']; + if (flutterVersion != null && frameworkRevision != null) { + return '$flutterVersion (revision $frameworkRevision)'; + } + return result.stdout.toString().trim(); + } + } catch (_) {} + + try { + final result = Process.runSync('flutter', [ + '--version', + ], runInShell: true); + if (result.exitCode == 0) { + return result.stdout.toString().trim().split('\n').first; + } + } catch (_) {} + + return '<unknown>'; + } + + String _detectDevToolsCommit() { + try { + final result = Process.runSync('git', [ + 'rev-parse', + 'HEAD', + ], runInShell: true); + if (result.exitCode == 0) { + return result.stdout.toString().trim(); + } + } catch (_) {} + return '<unknown>'; + } } Future<BenchmarkResults> runBenchmarkOrUseExisting(
diff --git a/packages/devtools_app/benchmark/scripts/utils.dart b/packages/devtools_app/benchmark/scripts/utils.dart index 6d7f24e..cb557c5 100644 --- a/packages/devtools_app/benchmark/scripts/utils.dart +++ b/packages/devtools_app/benchmark/scripts/utils.dart
@@ -45,14 +45,22 @@ extension BenchmarkScoreExtension on BenchmarkScore { List<String> toCsvLine() { + final deltaValue = delta; + final baselineValue = deltaValue != null ? value - deltaValue : null; + final String deltaPercent; + if (baselineValue == null) { + deltaPercent = ''; + } else if (baselineValue == 0) { + deltaPercent = 'N/A'; + } else { + deltaPercent = (deltaValue! / baselineValue).toString(); + } return [ metric, // Metric name - value.toString(), // Value - delta?.toString() ?? '', // Delta value - // value - delta represents the baseline score. - delta != null - ? (delta! / (value - delta!)).toString() - : '', // Delta % value + baselineValue?.toString() ?? '', // Baseline value + value.toString(), // Test value + deltaValue?.toString() ?? '', // Delta value + deltaPercent, // Delta % value ]; } }
diff --git a/packages/devtools_app/benchmark/test_infra/common.dart b/packages/devtools_app/benchmark/test_infra/common.dart index 32ab643..ec978f0 100644 --- a/packages/devtools_app/benchmark/test_infra/common.dart +++ b/packages/devtools_app/benchmark/test_infra/common.dart
@@ -10,11 +10,9 @@ /// found" in DevTools. const _benchmarkInitialPage = ''; -const _wasmQueryParameters = {'compiler': 'wasm'}; - String benchmarkPath({required bool useWasm}) => Uri( path: _benchmarkInitialPage, - queryParameters: useWasm ? _wasmQueryParameters : null, + queryParameters: {'compiler': useWasm ? 'wasm' : 'js'}, ).toString(); String generateBenchmarkEntryPoint({required bool useWasm}) {