blob: 0f37ad3918756d3195ee39f9a8f5de91e2f9640b [file] [log] [blame]
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:path/path.dart' as path;
import 'package:flutter_devicelab/framework/adb.dart';
import 'package:flutter_devicelab/framework/framework.dart';
import 'package:flutter_devicelab/framework/utils.dart';
/// Creates a device lab task that runs benchmarks in
/// `dev/benchmarks/microbenchmarks` reports results to the dashboard.
TaskFunction createMicrobenchmarkTask() {
return () async {
final Device device = await devices.workingDevice;
await device.unlock();
Future<Map<String, double>> _runMicrobench(String benchmarkPath) async {
Future<Map<String, double>> _run() async {
print('Running $benchmarkPath');
final Directory appDir = dir(
path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks'));
final Process flutterProcess = await inDirectory(appDir, () async {
final List<String> options = <String>[
// --release doesn't work on iOS due to code signing issues
return await _startFlutter(
options: options,
canFail: false,
return await _readJsonResults(flutterProcess);
return _run();
final Map<String, double> allResults = <String, double>{
...await _runMicrobench('lib/stocks/layout_bench.dart'),
...await _runMicrobench('lib/stocks/build_bench.dart'),
...await _runMicrobench('lib/geometry/matrix_utils_transform_bench.dart'),
...await _runMicrobench('lib/geometry/rrect_contains_bench.dart'),
...await _runMicrobench('lib/gestures/velocity_tracker_bench.dart'),
...await _runMicrobench('lib/gestures/gesture_detector_bench.dart'),
...await _runMicrobench('lib/stocks/animation_bench.dart'),
...await _runMicrobench('lib/language/sync_star_bench.dart'),
...await _runMicrobench('lib/language/sync_star_semantics_bench.dart'),
return TaskResult.success(allResults, benchmarkScoreKeys: allResults.keys.toList());
Future<Process> _startFlutter({
List<String> options = const <String>[],
bool canFail = false,
Map<String, String> environment,
}) {
final List<String> args = flutterCommandArgs('run', options);
return startProcess(path.join(flutterDirectory.path, 'bin', 'flutter'), args, environment: environment);
Future<Map<String, double>> _readJsonResults(Process process) {
// IMPORTANT: keep these values in sync with dev/benchmarks/microbenchmarks/lib/common.dart
const String jsonStart = '================ RESULTS ================';
const String jsonEnd = '================ FORMATTED ==============';
const String jsonPrefix = ':::JSON:::';
bool jsonStarted = false;
final StringBuffer jsonBuf = StringBuffer();
final Completer<Map<String, double>> completer = Completer<Map<String, double>>();
final StreamSubscription<String> stderrSub = process.stderr
.transform<String>(const Utf8Decoder())
.transform<String>(const LineSplitter())
.listen((String line) {
stderr.writeln('[STDERR] $line');
bool processWasKilledIntentionally = false;
bool resultsHaveBeenParsed = false;
final StreamSubscription<String> stdoutSub = process.stdout
.transform<String>(const Utf8Decoder())
.transform<String>(const LineSplitter())
.listen((String line) async {
if (line.contains(jsonStart)) {
jsonStarted = true;
if (line.contains(jsonEnd)) {
final String jsonOutput = jsonBuf.toString();
// If we end up here and have already parsed the results, it suggests that
// we have received output from another test because our `flutter run`
// process did not terminate correctly.
if (resultsHaveBeenParsed) {
throw 'Additional JSON was received after results has already been '
'processed. This suggests the `flutter run` process may have lived '
'past the end of our test and collected additional output from the '
'next test.\n\n'
'The JSON below contains all collected output, including both from '
'the original test and what followed.\n\n'
jsonStarted = false;
processWasKilledIntentionally = true;
resultsHaveBeenParsed = true;
// Sending a SIGINT/SIGTERM to the process here isn't reliable because [process] is
// the shell (flutter is a shell script) and doesn't pass the signal on.
// Sending a `q` is an instruction to quit using the console runner.
// See
await process.stdin.flush();
// Also send a kill signal in case the `q` above didn't work.
try {
completer.complete(Map<String, double>.from(json.decode(jsonOutput) as Map<String, dynamic>));
} catch (ex) {
completer.completeError('Decoding JSON failed ($ex). JSON string was: $jsonOutput');
if (jsonStarted && line.contains(jsonPrefix))
jsonBuf.writeln(line.substring(line.indexOf(jsonPrefix) + jsonPrefix.length));
process.exitCode.then<void>((int code) async {
await Future.wait<void>(<Future<void>>[
if (!processWasKilledIntentionally && code != 0) {
completer.completeError('flutter run failed: exit code=$code');
return completer.future;