blob: 9fdbcf0dc49cd165c956439c85d02948534fe3f6 [file] [log] [blame]
// Copyright 2016 The Chromium Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert' show json;
import 'dart:io';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/ios.dart';
import '../framework/utils.dart';
TaskFunction createComplexLayoutScrollPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
'test_driver/scroll_perf.dart',
'complex_layout_scroll_perf',
).run;
}
TaskFunction createTilesScrollPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
'test_driver/scroll_perf.dart',
'tiles_scroll_perf',
).run;
}
TaskFunction createHomeScrollPerfTest() {
return PerfTest(
'${flutterDirectory.path}/examples/flutter_gallery',
'test_driver/scroll_perf.dart',
'home_scroll_perf',
).run;
}
TaskFunction createCullOpacityPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/cull_opacity_perf.dart',
'cull_opacity_perf',
).run;
}
TaskFunction createCubicBezierPerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/cubic_bezier_perf.dart',
'cubic_bezier_perf',
).run;
}
TaskFunction createBackdropFilterPerfTest({bool needsMeasureCpuGpu = false}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/backdrop_filter_perf.dart',
'backdrop_filter_perf',
needsMeasureCpuGPu: needsMeasureCpuGpu,
).run;
}
TaskFunction createSimpleAnimationPerfTest({bool needsMeasureCpuGpu = false}) {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/simple_animation_perf.dart',
'simple_animation_perf',
needsMeasureCpuGPu: needsMeasureCpuGpu,
).run;
}
TaskFunction createPictureCachePerfTest() {
return PerfTest(
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks',
'test_driver/picture_cache_perf.dart',
'picture_cache_perf',
).run;
}
TaskFunction createFlutterGalleryStartupTest() {
return StartupTest(
'${flutterDirectory.path}/examples/flutter_gallery',
).run;
}
TaskFunction createComplexLayoutStartupTest() {
return StartupTest(
'${flutterDirectory.path}/dev/benchmarks/complex_layout',
).run;
}
TaskFunction createHelloWorldStartupTest() {
return StartupTest(
'${flutterDirectory.path}/examples/hello_world',
reportMetrics: false,
).run;
}
TaskFunction createFlutterGalleryCompileTest() {
return CompileTest('${flutterDirectory.path}/examples/flutter_gallery').run;
}
TaskFunction createHelloWorldCompileTest() {
return CompileTest('${flutterDirectory.path}/examples/hello_world', reportPackageContentSizes: true).run;
}
TaskFunction createWebCompileTest() {
return const WebCompileTest().run;
}
TaskFunction createComplexLayoutCompileTest() {
return CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
}
TaskFunction createFlutterViewStartupTest() {
return StartupTest(
'${flutterDirectory.path}/examples/flutter_view',
reportMetrics: false,
).run;
}
TaskFunction createPlatformViewStartupTest() {
return StartupTest(
'${flutterDirectory.path}/examples/platform_view',
reportMetrics: false,
).run;
}
TaskFunction createBasicMaterialCompileTest() {
return () async {
const String sampleAppName = 'sample_flutter_app';
final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');
rmTree(sampleDir);
await inDirectory<void>(Directory.systemTemp, () async {
await flutter('create', options: <String>['--template=app', sampleAppName]);
});
if (!sampleDir.existsSync())
throw 'Failed to create default Flutter app in ${sampleDir.path}';
return CompileTest(sampleDir.path).run();
};
}
/// Measure application startup performance.
class StartupTest {
const StartupTest(this.testDirectory, { this.reportMetrics = true });
final String testDirectory;
final bool reportMetrics;
Future<TaskResult> run() async {
return await inDirectory<TaskResult>(testDirectory, () async {
final String deviceId = (await devices.workingDevice).deviceId;
await flutter('packages', options: <String>['get']);
await flutter('run', options: <String>[
'--verbose',
'--profile',
'--trace-startup',
'-d',
deviceId,
]);
final Map<String, dynamic> data = json.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync());
if (!reportMetrics)
return TaskResult.success(data);
return TaskResult.success(data, benchmarkScoreKeys: <String>[
'timeToFirstFrameMicros',
'timeToFirstFrameRasterizedMicros',
]);
});
}
}
/// Measures application runtime performance, specifically per-frame
/// performance.
class PerfTest {
const PerfTest(
this.testDirectory,
this.testTarget,
this.timelineFileName,
{this.needsMeasureCpuGPu = false});
final String testDirectory;
final String testTarget;
final String timelineFileName;
final bool needsMeasureCpuGPu;
Future<TaskResult> run() {
return inDirectory<TaskResult>(testDirectory, () async {
final Device device = await devices.workingDevice;
await device.unlock();
final String deviceId = device.deviceId;
await flutter('packages', options: <String>['get']);
await flutter('drive', options: <String>[
'-v',
'--profile',
'--trace-startup', // Enables "endless" timeline event buffering.
'-t',
testTarget,
'-d',
deviceId,
]);
final Map<String, dynamic> data = json.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync());
if (data['frame_count'] < 5) {
return TaskResult.failure(
'Timeline contains too few frames: ${data['frame_count']}. Possibly '
'trace events are not being captured.',
);
}
if (needsMeasureCpuGPu) {
await inDirectory<void>('$testDirectory/build', () async {
data.addAll(await measureIosCpuGpu(deviceId: deviceId));
});
}
return TaskResult.success(data, benchmarkScoreKeys: <String>[
'average_frame_build_time_millis',
'worst_frame_build_time_millis',
'missed_frame_build_budget_count',
'90th_percentile_frame_build_time_millis',
'99th_percentile_frame_build_time_millis',
'average_frame_rasterizer_time_millis',
'worst_frame_rasterizer_time_millis',
'missed_frame_rasterizer_budget_count',
'90th_percentile_frame_rasterizer_time_millis',
'99th_percentile_frame_rasterizer_time_millis',
if (needsMeasureCpuGPu) 'cpu_percentage',
if (needsMeasureCpuGPu) 'gpu_percentage',
]);
});
}
}
/// Measures how long it takes to compile a Flutter app to JavaScript and how
/// big the compiled code is.
class WebCompileTest {
const WebCompileTest();
Future<TaskResult> run() async {
final Map<String, Object> metrics = <String, Object>{};
await inDirectory<TaskResult>('${flutterDirectory.path}/examples/hello_world', () async {
await flutter('packages', options: <String>['get']);
await evalFlutter('build', options: <String>[
'web',
'-v',
'--release',
'--no-pub',
], environment: <String, String>{
'FLUTTER_WEB': 'true',
});
final String output = '${flutterDirectory.path}/examples/hello_world/build/web/main.dart.js';
await _measureSize('hello_world', output, metrics);
return null;
});
await inDirectory<TaskResult>('${flutterDirectory.path}/examples/flutter_gallery', () async {
await flutter('packages', options: <String>['get']);
await evalFlutter('build', options: <String>[
'web',
'-v',
'--release',
'--no-pub',
], environment: <String, String>{
'FLUTTER_WEB': 'true',
});
final String output = '${flutterDirectory.path}/examples/flutter_gallery/build/web/main.dart.js';
await _measureSize('flutter_gallery', output, metrics);
return null;
});
const String sampleAppName = 'sample_flutter_app';
final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName');
rmTree(sampleDir);
await inDirectory<void>(Directory.systemTemp, () async {
await flutter('create', options: <String>['--template=app', '--web', sampleAppName], environment: <String, String>{
'FLUTTER_WEB': 'true',
});
await inDirectory(sampleDir, () async {
await flutter('packages', options: <String>['get']);
await evalFlutter('build', options: <String>[
'web',
'-v',
'--release',
'--no-pub',
], environment: <String, String>{
'FLUTTER_WEB': 'true',
});
await _measureSize('basic_material_app', path.join(sampleDir.path, 'build/web/main.dart.js'), metrics);
});
});
return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
}
static Future<void> _measureSize(String metric, String output, Map<String, Object> metrics) async {
final ProcessResult result = await Process.run('du', <String>['-k', output]);
await Process.run('gzip',<String>['-k', '9', output]);
final ProcessResult resultGzip = await Process.run('du', <String>['-k', output + '.gz']);
metrics['${metric}_dart2js_size'] = _parseDu(result.stdout);
metrics['${metric}_dart2js_size_gzip'] = _parseDu(resultGzip.stdout);
}
static int _parseDu(String source) {
return int.parse(source.split(RegExp(r'\s+')).first.trim());
}
}
/// Measures how long it takes to compile a Flutter app and how big the compiled
/// code is.
class CompileTest {
const CompileTest(this.testDirectory, { this.reportPackageContentSizes = false });
final String testDirectory;
final bool reportPackageContentSizes;
Future<TaskResult> run() async {
return await inDirectory<TaskResult>(testDirectory, () async {
final Device device = await devices.workingDevice;
await device.unlock();
await flutter('packages', options: <String>['get']);
final Map<String, dynamic> metrics = <String, dynamic>{
...await _compileAot(),
...await _compileApp(reportPackageContentSizes: reportPackageContentSizes),
...await _compileDebug(),
};
return TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList());
});
}
static Future<Map<String, dynamic>> _compileAot() async {
await flutter('clean');
final Stopwatch watch = Stopwatch()..start();
final List<String> options = <String>[
'aot',
'-v',
'--extra-gen-snapshot-options=--print_snapshot_sizes',
'--release',
'--no-pub',
'--target-platform',
];
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.ios:
options.add('ios');
break;
case DeviceOperatingSystem.android:
options.add('android-arm');
break;
}
final String compileLog = await evalFlutter('build', options: options);
watch.stop();
final RegExp metricExpression = RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)');
final Map<String, dynamic> metrics = <String, dynamic>{};
for (Match m in metricExpression.allMatches(compileLog)) {
metrics[_sdkNameToMetricName(m.group(1))] = int.parse(m.group(2));
}
if (metrics.length != _kSdkNameToMetricNameMapping.length) {
throw 'Expected metrics: ${_kSdkNameToMetricNameMapping.keys}, but got: ${metrics.keys}.';
}
metrics['aot_snapshot_compile_millis'] = watch.elapsedMilliseconds;
return metrics;
}
static Future<Map<String, dynamic>> _compileApp({ bool reportPackageContentSizes = false }) async {
await flutter('clean');
final Stopwatch watch = Stopwatch();
int releaseSizeInBytes;
final List<String> options = <String>['--release'];
final Map<String, dynamic> metrics = <String, dynamic>{};
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.ios:
options.insert(0, 'ios');
watch.start();
await flutter('build', options: options);
watch.stop();
final String appPath = '$cwd/build/ios/Release-iphoneos/Runner.app/';
// IPAs are created manually, https://flutter.dev/ios-release/
await exec('tar', <String>['-zcf', 'build/app.ipa', appPath]);
releaseSizeInBytes = await file('$cwd/build/app.ipa').length();
if (reportPackageContentSizes)
metrics.addAll(await getSizesFromIosApp(appPath));
break;
case DeviceOperatingSystem.android:
options.insert(0, 'apk');
options.add('--target-platform=android-arm');
watch.start();
await flutter('build', options: options);
watch.stop();
String apkPath = '$cwd/build/app/outputs/apk/app.apk';
File apk = file(apkPath);
if (!apk.existsSync()) {
// Pre Android SDK 26 path
apkPath = '$cwd/build/app/outputs/apk/app-release.apk';
apk = file(apkPath);
}
releaseSizeInBytes = apk.lengthSync();
if (reportPackageContentSizes)
metrics.addAll(await getSizesFromApk(apkPath));
break;
}
metrics.addAll(<String, dynamic>{
'release_full_compile_millis': watch.elapsedMilliseconds,
'release_size_bytes': releaseSizeInBytes,
});
return metrics;
}
static Future<Map<String, dynamic>> _compileDebug() async {
await flutter('clean');
final Stopwatch watch = Stopwatch();
final List<String> options = <String>['--debug'];
switch (deviceOperatingSystem) {
case DeviceOperatingSystem.ios:
options.insert(0, 'ios');
break;
case DeviceOperatingSystem.android:
options.insert(0, 'apk');
options.add('--target-platform=android-arm');
break;
}
watch.start();
await flutter('build', options: options);
watch.stop();
return <String, dynamic>{
'debug_full_compile_millis': watch.elapsedMilliseconds,
};
}
static const Map<String, String> _kSdkNameToMetricNameMapping = <String, String> {
'VMIsolate': 'aot_snapshot_size_vmisolate',
'Isolate': 'aot_snapshot_size_isolate',
'ReadOnlyData': 'aot_snapshot_size_rodata',
'Instructions': 'aot_snapshot_size_instructions',
'Total': 'aot_snapshot_size_total',
};
static String _sdkNameToMetricName(String sdkName) {
if (!_kSdkNameToMetricNameMapping.containsKey(sdkName))
throw 'Unrecognized SDK snapshot metric name: $sdkName';
return _kSdkNameToMetricNameMapping[sdkName];
}
static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
// Thin the binary to only contain one architecture.
final String xcodeBackend = path.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
'ARCHS': 'arm64',
'WRAPPER_NAME': path.basename(appPath),
'TARGET_BUILD_DIR': path.dirname(appPath),
});
final File appFramework = File(path.join(appPath, 'Frameworks', 'App.framework', 'App'));
final File flutterFramework = File(path.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));
return <String, dynamic>{
'app_framework_uncompressed_bytes': await appFramework.length(),
'flutter_framework_uncompressed_bytes': await flutterFramework.length(),
};
}
static Future<Map<String, dynamic>> getSizesFromApk(String apkPath) async {
final String output = await eval('unzip', <String>['-v', apkPath]);
final List<String> lines = output.split('\n');
final Map<String, _UnzipListEntry> fileToMetadata = <String, _UnzipListEntry>{};
// First three lines are header, last two lines are footer.
for (int i = 3; i < lines.length - 2; i++) {
final _UnzipListEntry entry = _UnzipListEntry.fromLine(lines[i]);
fileToMetadata[entry.path] = entry;
}
final _UnzipListEntry libflutter = fileToMetadata['lib/armeabi-v7a/libflutter.so'];
final _UnzipListEntry libapp = fileToMetadata['lib/armeabi-v7a/libapp.so'];
final _UnzipListEntry license = fileToMetadata['assets/flutter_assets/LICENSE'];
return <String, dynamic>{
'libflutter_uncompressed_bytes': libflutter.uncompressedSize,
'libflutter_compressed_bytes': libflutter.compressedSize,
'libapp_uncompressed_bytes': libapp.uncompressedSize,
'libapp_compressed_bytes': libapp.compressedSize,
'license_uncompressed_bytes': license.uncompressedSize,
'license_compressed_bytes': license.compressedSize,
};
}
}
/// Measure application memory usage.
class MemoryTest {
MemoryTest(this.project, this.test, this.package);
final String project;
final String test;
final String package;
/// Completes when the log line specified in the last call to
/// [prepareForNextMessage] is seen by `adb logcat`.
Future<void> get receivedNextMessage => _receivedNextMessage?.future;
Completer<void> _receivedNextMessage;
String _nextMessage;
/// Prepares the [receivedNextMessage] future such that it will complete
/// when `adb logcat` sees a log line with the given `message`.
void prepareForNextMessage(String message) {
_nextMessage = message;
_receivedNextMessage = Completer<void>();
}
int get iterationCount => 10;
Device get device => _device;
Device _device;
Future<TaskResult> run() {
return inDirectory<TaskResult>(project, () async {
// This test currently only works on Android, because device.logcat,
// device.getMemoryStats, etc, aren't implemented for iOS.
_device = await devices.workingDevice;
await device.unlock();
await flutter('packages', options: <String>['get']);
final StreamSubscription<String> adb = device.logcat.listen(
(String data) {
if (data.contains('==== MEMORY BENCHMARK ==== $_nextMessage ===='))
_receivedNextMessage.complete();
},
);
for (int iteration = 0; iteration < iterationCount; iteration += 1) {
print('running memory test iteration $iteration...');
_startMemoryUsage = null;
await useMemory();
assert(_startMemoryUsage != null);
assert(_startMemory.length == iteration + 1);
assert(_endMemory.length == iteration + 1);
assert(_diffMemory.length == iteration + 1);
print('terminating...');
await device.stop(package);
await Future<void>.delayed(const Duration(milliseconds: 10));
}
await adb.cancel();
final ListStatistics startMemoryStatistics = ListStatistics(_startMemory);
final ListStatistics endMemoryStatistics = ListStatistics(_endMemory);
final ListStatistics diffMemoryStatistics = ListStatistics(_diffMemory);
final Map<String, dynamic> memoryUsage = <String, dynamic>{
...startMemoryStatistics.asMap('start'),
...endMemoryStatistics.asMap('end'),
...diffMemoryStatistics.asMap('diff'),
};
_device = null;
_startMemory.clear();
_endMemory.clear();
_diffMemory.clear();
return TaskResult.success(memoryUsage, benchmarkScoreKeys: memoryUsage.keys.toList());
});
}
/// Starts the app specified by [test] on the [device].
///
/// The [run] method will terminate it by its package name ([package]).
Future<void> launchApp() async {
prepareForNextMessage('READY');
print('launching $project$test on device...');
await flutter('run', options: <String>[
'--verbose',
'--release',
'--no-resident',
'-d', device.deviceId,
test,
]);
print('awaiting "ready" message...');
await receivedNextMessage;
}
/// To change the behavior of the test, override this.
///
/// Make sure to call recordStart() and recordEnd() once each in that order.
///
/// By default it just launches the app, records memory usage, taps the device,
/// awaits a DONE notification, and records memory usage again.
Future<void> useMemory() async {
await launchApp();
await recordStart();
prepareForNextMessage('DONE');
print('tapping device...');
await device.tap(100, 100);
print('awaiting "done" message...');
await receivedNextMessage;
await recordEnd();
}
final List<int> _startMemory = <int>[];
final List<int> _endMemory = <int>[];
final List<int> _diffMemory = <int>[];
Map<String, dynamic> _startMemoryUsage;
@protected
Future<void> recordStart() async {
assert(_startMemoryUsage == null);
print('snapshotting memory usage...');
_startMemoryUsage = await device.getMemoryStats(package);
}
@protected
Future<void> recordEnd() async {
assert(_startMemoryUsage != null);
print('snapshotting memory usage...');
final Map<String, dynamic> endMemoryUsage = await device.getMemoryStats(package);
_startMemory.add(_startMemoryUsage['total_kb']);
_endMemory.add(endMemoryUsage['total_kb']);
_diffMemory.add(endMemoryUsage['total_kb'] - _startMemoryUsage['total_kb']);
}
}
enum ReportedDurationTestFlavor {
debug, profile, release
}
String _reportedDurationTestToString(ReportedDurationTestFlavor flavor) {
switch (flavor) {
case ReportedDurationTestFlavor.debug:
return 'debug';
case ReportedDurationTestFlavor.profile:
return 'profile';
case ReportedDurationTestFlavor.release:
return 'release';
}
throw ArgumentError('Unexpected value for enum $flavor');
}
class ReportedDurationTest {
ReportedDurationTest(this.flavor, this.project, this.test, this.package, this.durationPattern);
final ReportedDurationTestFlavor flavor;
final String project;
final String test;
final String package;
final RegExp durationPattern;
final Completer<int> durationCompleter = Completer<int>();
int get iterationCount => 10;
Device get device => _device;
Device _device;
Future<TaskResult> run() {
return inDirectory<TaskResult>(project, () async {
// This test currently only works on Android, because device.logcat,
// device.getMemoryStats, etc, aren't implemented for iOS.
_device = await devices.workingDevice;
await device.unlock();
await flutter('packages', options: <String>['get']);
final StreamSubscription<String> adb = device.logcat.listen(
(String data) {
if (durationPattern.hasMatch(data))
durationCompleter.complete(int.parse(durationPattern.firstMatch(data).group(1)));
},
);
print('launching $project$test on device...');
await flutter('run', options: <String>[
'--verbose',
'--${_reportedDurationTestToString(flavor)}',
'--no-resident',
'-d', device.deviceId,
test,
]);
final int duration = await durationCompleter.future;
print('terminating...');
await device.stop(package);
await adb.cancel();
_device = null;
final Map<String, dynamic> reportedDuration = <String, dynamic>{
'duration': duration,
};
_device = null;
return TaskResult.success(reportedDuration, benchmarkScoreKeys: reportedDuration.keys.toList());
});
}
}
/// Holds simple statistics of an odd-lengthed list of integers.
class ListStatistics {
factory ListStatistics(Iterable<int> data) {
assert(data.isNotEmpty);
assert(data.length % 2 == 1);
final List<int> sortedData = data.toList()..sort();
return ListStatistics._(
sortedData.first,
sortedData.last,
sortedData[(sortedData.length - 1) ~/ 2],
);
}
const ListStatistics._(this.min, this.max, this.median);
final int min;
final int max;
final int median;
Map<String, int> asMap(String prefix) {
return <String, int>{
'$prefix-min': min,
'$prefix-max': max,
'$prefix-median': median,
};
}
}
class _UnzipListEntry {
factory _UnzipListEntry.fromLine(String line) {
final List<String> data = line.trim().split(RegExp('\\s+'));
assert(data.length == 8);
return _UnzipListEntry._(
uncompressedSize: int.parse(data[0]),
compressedSize: int.parse(data[2]),
path: data[7],
);
}
_UnzipListEntry._({
@required this.uncompressedSize,
@required this.compressedSize,
@required this.path,
}) : assert(uncompressedSize != null),
assert(compressedSize != null),
assert(compressedSize <= uncompressedSize),
assert(path != null);
final int uncompressedSize;
final int compressedSize;
final String path;
}