A benchmark test case for measuring scroll smoothness (#61998)
diff --git a/dev/benchmarks/complex_layout/android/settings.gradle b/dev/benchmarks/complex_layout/android/settings.gradle
index 663db3a..d3b6a40 100644
--- a/dev/benchmarks/complex_layout/android/settings.gradle
+++ b/dev/benchmarks/complex_layout/android/settings.gradle
@@ -3,3 +3,13 @@
// found in the LICENSE file.
include ':app'
+
+def localPropertiesFile = new File(rootProject.projectDir, "local.properties")
+def properties = new Properties()
+
+assert localPropertiesFile.exists()
+localPropertiesFile.withReader("UTF-8") { reader -> properties.load(reader) }
+
+def flutterSdkPath = properties.getProperty("flutter.sdk")
+assert flutterSdkPath != null, "flutter.sdk not set in local.properties"
+apply from: "$flutterSdkPath/packages/flutter_tools/gradle/app_plugin_loader.gradle"
diff --git a/dev/benchmarks/complex_layout/lib/main.dart b/dev/benchmarks/complex_layout/lib/main.dart
index 4496bdd..69d6b13 100644
--- a/dev/benchmarks/complex_layout/lib/main.dart
+++ b/dev/benchmarks/complex_layout/lib/main.dart
@@ -109,6 +109,7 @@
Expanded(
child: ListView.builder(
key: const Key('complex-scroll'), // this key is used by the driver test
+ controller: ScrollController(), // So that the scroll offset can be tracked
itemBuilder: (BuildContext context, int index) {
if (index % 2 == 0)
return FancyImageItem(index, key: PageStorageKey<int>(index));
diff --git a/dev/benchmarks/complex_layout/pubspec.yaml b/dev/benchmarks/complex_layout/pubspec.yaml
index 8d20470..8d52efd 100644
--- a/dev/benchmarks/complex_layout/pubspec.yaml
+++ b/dev/benchmarks/complex_layout/pubspec.yaml
@@ -3,7 +3,7 @@
environment:
# The pub client defaults to an <2.0.0 sdk constraint which we need to explicitly overwrite.
- sdk: ">=2.0.0-dev.68.0 <3.0.0"
+ sdk: ">=2.2.2 <3.0.0"
dependencies:
flutter:
@@ -46,6 +46,7 @@
flutter_test:
sdk: flutter
test: 1.16.0-nullsafety.1
+ e2e: 0.7.0
_fe_analyzer_shared: 7.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
analyzer: 0.39.17 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -90,4 +91,4 @@
- packages/flutter_gallery_assets/people/square/ali.png
- packages/flutter_gallery_assets/places/india_chettinad_silk_maker.png
-# PUBSPEC CHECKSUM: 6832
+# PUBSPEC CHECKSUM: 047d
diff --git a/dev/benchmarks/complex_layout/test/measure_scroll_smoothness.dart b/dev/benchmarks/complex_layout/test/measure_scroll_smoothness.dart
new file mode 100644
index 0000000..05c5107
--- /dev/null
+++ b/dev/benchmarks/complex_layout/test/measure_scroll_smoothness.dart
@@ -0,0 +1,296 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// This test is a use case of flutter/flutter#60796
+// the test should be run as:
+// flutter drive -t test/using_array.dart --driver test_driver/scrolling_test_e2e_test.dart
+
+import 'dart:ui' as ui;
+
+import 'package:flutter/gestures.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter_test/flutter_test.dart';
+import 'package:e2e/e2e.dart';
+
+import 'package:complex_layout/main.dart' as app;
+
+class PointerDataTestBinding extends E2EWidgetsFlutterBinding {
+ // PointerData injection would usually be considered device input and therefore
+ // blocked by [TestWidgetsFlutterBinding]. Override this behavior
+ // to help events go into widget tree.
+ @override
+ void dispatchEvent(
+ PointerEvent event,
+ HitTestResult hitTestResult, {
+ TestBindingEventSource source = TestBindingEventSource.device,
+ }) {
+ super.dispatchEvent(event, hitTestResult, source: TestBindingEventSource.test);
+ }
+}
+
+/// A union of [ui.PointerDataPacket] and the time it should be sent.
+class PointerDataRecord {
+ PointerDataRecord(this.timeStamp, List<ui.PointerData> data)
+ : data = ui.PointerDataPacket(data: data);
+ final ui.PointerDataPacket data;
+ final Duration timeStamp;
+}
+
+/// Generates the [PointerDataRecord] to simulate a drag operation from
+/// `center - totalMove/2` to `center + totalMove/2`.
+Iterable<PointerDataRecord> dragInputDatas(
+ final Duration epoch,
+ final Offset center, {
+ final Offset totalMove = const Offset(0, -400),
+ final Duration totalTime = const Duration(milliseconds: 2000),
+ final double frequency = 90,
+}) sync* {
+ final Offset startLocation = (center - totalMove / 2) * ui.window.devicePixelRatio;
+ // The issue is about 120Hz input on 90Hz refresh rate device.
+ // We test 90Hz input on 60Hz device here, which shows similar pattern.
+ final int moveEventCount = totalTime.inMicroseconds * frequency ~/ const Duration(seconds: 1).inMicroseconds;
+ final Offset movePerEvent = totalMove / moveEventCount.toDouble() * ui.window.devicePixelRatio;
+ yield PointerDataRecord(epoch, <ui.PointerData>[
+ ui.PointerData(
+ timeStamp: epoch,
+ change: ui.PointerChange.add,
+ physicalX: startLocation.dx,
+ physicalY: startLocation.dy,
+ ),
+ ui.PointerData(
+ timeStamp: epoch,
+ change: ui.PointerChange.down,
+ physicalX: startLocation.dx,
+ physicalY: startLocation.dy,
+ pointerIdentifier: 1,
+ ),
+ ]);
+ for (int t = 0; t < moveEventCount + 1; t++) {
+ final Offset position = startLocation + movePerEvent * t.toDouble();
+ yield PointerDataRecord(
+ epoch + totalTime * t ~/ moveEventCount,
+ <ui.PointerData>[ui.PointerData(
+ timeStamp: epoch + totalTime * t ~/ moveEventCount,
+ change: ui.PointerChange.move,
+ physicalX: position.dx,
+ physicalY: position.dy,
+ // Scrolling behavior depends on this delta rather
+ // than the position difference.
+ physicalDeltaX: movePerEvent.dx,
+ physicalDeltaY: movePerEvent.dy,
+ pointerIdentifier: 1,
+ )],
+ );
+ }
+ final Offset position = startLocation + totalMove;
+ yield PointerDataRecord(epoch + totalTime, <ui.PointerData>[ui.PointerData(
+ timeStamp: epoch + totalTime,
+ change: ui.PointerChange.up,
+ physicalX: position.dx,
+ physicalY: position.dy,
+ pointerIdentifier: 1,
+ )]);
+}
+
+enum TestScenario {
+ resampleOn90Hz,
+ resampleOn59Hz,
+ resampleOff90Hz,
+ resampleOff59Hz,
+}
+
+class ResampleFlagVariant extends TestVariant<TestScenario> {
+ ResampleFlagVariant(this.binding);
+ final E2EWidgetsFlutterBinding binding;
+
+ @override
+ final Set<TestScenario> values = Set<TestScenario>.from(TestScenario.values);
+
+ TestScenario currentValue;
+ bool get resample {
+ switch(currentValue) {
+ case TestScenario.resampleOn90Hz:
+ case TestScenario.resampleOn59Hz:
+ return true;
+ case TestScenario.resampleOff90Hz:
+ case TestScenario.resampleOff59Hz:
+ return false;
+ }
+ throw ArgumentError;
+ }
+ double get frequency {
+ switch(currentValue) {
+ case TestScenario.resampleOn90Hz:
+ case TestScenario.resampleOff90Hz:
+ return 90.0;
+ case TestScenario.resampleOn59Hz:
+ case TestScenario.resampleOff59Hz:
+ return 59.0;
+ }
+ throw ArgumentError;
+ }
+
+ Map<String, dynamic> result;
+
+ @override
+ String describeValue(TestScenario value) {
+ switch(value) {
+ case TestScenario.resampleOn90Hz:
+ return 'resample on with 90Hz input';
+ case TestScenario.resampleOn59Hz:
+ return 'resample on with 59Hz input';
+ case TestScenario.resampleOff90Hz:
+ return 'resample off with 90Hz input';
+ case TestScenario.resampleOff59Hz:
+ return 'resample off with 59Hz input';
+ }
+ throw ArgumentError;
+ }
+
+ @override
+ Future<bool> setUp(TestScenario value) async {
+ currentValue = value;
+ final bool original = binding.resamplingEnabled;
+ binding.resamplingEnabled = resample;
+ return original;
+ }
+
+ @override
+ Future<void> tearDown(TestScenario value, bool memento) async {
+ binding.resamplingEnabled = memento;
+ binding.reportData[describeValue(value)] = result;
+ }
+}
+
+Future<void> main() async {
+ final PointerDataTestBinding binding = PointerDataTestBinding();
+ assert(WidgetsBinding.instance == binding);
+ binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive;
+ binding.reportData ??= <String, dynamic>{};
+ final ResampleFlagVariant variant = ResampleFlagVariant(binding);
+ testWidgets('Smoothness test', (WidgetTester tester) async {
+ app.main();
+ await tester.pumpAndSettle();
+ final Finder scrollerFinder = find.byKey(const ValueKey<String>('complex-scroll'));
+ final ListView scroller = tester.widget<ListView>(scrollerFinder);
+ final ScrollController controller = scroller.controller;
+ final List<int> frameTimestamp = <int>[];
+ final List<double> scrollOffset = <double>[];
+ final List<Duration> delays = <Duration>[];
+ binding.addPersistentFrameCallback((Duration timeStamp) {
+ if (controller.hasClients) {
+ // This if is necessary because by the end of the test the widget tree
+ // is destroyed.
+ frameTimestamp.add(timeStamp.inMicroseconds);
+ scrollOffset.add(controller.offset);
+ }
+ });
+
+ Duration now() => binding.currentSystemFrameTimeStamp;
+ Future<void> scroll() async {
+ // Extra 50ms to avoid timeouts.
+ final Duration startTime = const Duration(milliseconds: 500) + now();
+ for (final PointerDataRecord record in dragInputDatas(
+ startTime,
+ tester.getCenter(scrollerFinder),
+ frequency: variant.frequency,
+ )) {
+ await tester.binding.delayed(record.timeStamp - now());
+ // This now measures how accurate the above delayed is.
+ final Duration delay = now() - record.timeStamp;
+ if (delays.length < frameTimestamp.length) {
+ while (delays.length < frameTimestamp.length - 1) {
+ delays.add(Duration.zero);
+ }
+ delays.add(delay);
+ } else if (delays.last < delay) {
+ delays.last = delay;
+ }
+ ui.window.onPointerDataPacket(record.data);
+ }
+ }
+
+ for (int n = 0; n < 5; n++) {
+ await scroll();
+ }
+ variant.result = scrollSummary(scrollOffset, delays, frameTimestamp);
+ await tester.pumpAndSettle();
+ scrollOffset.clear();
+ delays.clear();
+ await tester.idle();
+ }, semanticsEnabled: false, variant: variant);
+}
+
+/// Calculates the smoothness measure from `scrollOffset` and `delays` list.
+///
+/// Smoothness (`abs_jerk`) is measured by the absolute value of the discrete
+/// 2nd derivative of the scroll offset.
+///
+/// It was experimented that jerk (3rd derivative of the position) is a good
+/// measure the smoothness.
+/// Here we are using 2nd derivative instead because the input is completely
+/// linear and the expected acceleration should be strictly zero.
+/// Observed acceleration is jumping from positive to negative within
+/// adjacent frames, meaning mathematically the discrete 3-rd derivative
+/// (`f[3] - 3*f[2] + 3*f[1] - f[0]`) is not a good approximation of jerk
+/// (continuous 3-rd derivative), while discrete 2nd
+/// derivative (`f[2] - 2*f[1] + f[0]`) on the other hand is a better measure
+/// of how the scrolling deviate away from linear, and given the acceleration
+/// should average to zero within two frames, it's also a good approximation
+/// for jerk in terms of physics.
+/// We use abs rather than square because square (2-norm) amplifies the
+/// effect of the data point that's relatively large, but in this metric
+/// we prefer smaller data point to have similar effect.
+/// This is also why we count the number of data that's larger than a
+/// threshold (and the result is tested not sensitive to this threshold),
+/// which is effectively a 0-norm.
+///
+/// Frames that are too slow to build (longer than 40ms) or with input delay
+/// longer than 16ms (1/60Hz) is filtered out to separate the janky due to slow
+/// response.
+///
+/// The returned map has keys:
+/// `average_abs_jerk`: average for the overall smoothness.
+/// `janky_count`: number of frames with `abs_jerk` larger than 0.5.
+/// `dropped_frame_count`: number of frames that are built longer than 40ms and
+/// are not used for smoothness measurement.
+/// `frame_timestamp`: the list of the timestamp for each frame, in the time
+/// order.
+/// `scroll_offset`: the scroll offset for each frame. Its length is the same as
+/// `frame_timestamp`.
+/// `input_delay`: the list of maximum delay time of the input simulation during
+/// a frame. Its length is the same as `frame_timestamp`
+Map<String, dynamic> scrollSummary(
+ List<double> scrollOffset,
+ List<Duration> delays,
+ List<int> frameTimestamp,
+) {
+ double jankyCount = 0;
+ double absJerkAvg = 0;
+ int lostFrame = 0;
+ for (int i = 1; i < scrollOffset.length-1; i += 1) {
+ if (frameTimestamp[i+1] - frameTimestamp[i-1] > 40E3 ||
+ (i >= delays.length || delays[i] > const Duration(milliseconds: 16))) {
+ // filter data points from slow frame building or input simulation artifact
+ lostFrame += 1;
+ continue;
+ }
+ //
+ final double absJerk = (scrollOffset[i-1] + scrollOffset[i+1] - 2*scrollOffset[i]).abs();
+ absJerkAvg += absJerk;
+ if (absJerk > 0.5)
+ jankyCount += 1;
+ }
+ // expect(lostFrame < 0.1 * frameTimestamp.length, true);
+ absJerkAvg /= frameTimestamp.length - lostFrame;
+
+ return <String, dynamic>{
+ 'janky_count': jankyCount,
+ 'average_abs_jerk': absJerkAvg,
+ 'dropped_frame_count': lostFrame,
+ 'frame_timestamp': List<int>.from(frameTimestamp),
+ 'scroll_offset': List<double>.from(scrollOffset),
+ 'input_delay': delays.map<int>((Duration data) => data.inMicroseconds).toList(),
+ };
+}
diff --git a/dev/benchmarks/complex_layout/test_driver/measure_scroll_smoothness_test.dart b/dev/benchmarks/complex_layout/test_driver/measure_scroll_smoothness_test.dart
new file mode 100644
index 0000000..fd29ebe
--- /dev/null
+++ b/dev/benchmarks/complex_layout/test_driver/measure_scroll_smoothness_test.dart
@@ -0,0 +1,17 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:e2e/e2e_driver.dart' as driver;
+
+Future<void> main() => driver.e2eDriver(
+ timeout: const Duration(minutes: 5),
+ responseDataCallback: (Map<String, dynamic> data) async {
+ await driver.writeResponseData(
+ data,
+ testOutputFilename: 'scroll_smoothness_test',
+ );
+ }
+);
diff --git a/dev/devicelab/bin/tasks/complex_layout_android__scroll_smoothness.dart b/dev/devicelab/bin/tasks/complex_layout_android__scroll_smoothness.dart
new file mode 100644
index 0000000..960965e
--- /dev/null
+++ b/dev/devicelab/bin/tasks/complex_layout_android__scroll_smoothness.dart
@@ -0,0 +1,14 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+import 'package:flutter_devicelab/framework/framework.dart';
+
+Future<void> main() async {
+ deviceOperatingSystem = DeviceOperatingSystem.android;
+ await task(createsScrollSmoothnessPerfTest());
+}
diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart
index b02975c..a62b2ed 100644
--- a/dev/devicelab/lib/tasks/perf_tests.dart
+++ b/dev/devicelab/lib/tasks/perf_tests.dart
@@ -298,6 +298,54 @@
).run;
}
+TaskFunction createsScrollSmoothnessPerfTest() {
+ final String testDirectory =
+ '${flutterDirectory.path}/dev/benchmarks/complex_layout';
+ const String testTarget = 'test/measure_scroll_smoothness.dart';
+ return () {
+ return inDirectory<TaskResult>(testDirectory, () async {
+ final Device device = await devices.workingDevice;
+ await device.unlock();
+ final String deviceId = device.deviceId;
+ await flutter('packages', options: <String>['get']);
+
+ await flutter('drive', options: <String>[
+ '-v',
+ '--verbose-system-logs',
+ '--profile',
+ '-t', testTarget,
+ '-d',
+ deviceId,
+ ]);
+ final Map<String, dynamic> data = json.decode(
+ file('$testDirectory/build/scroll_smoothness_test.json').readAsStringSync(),
+ ) as Map<String, dynamic>;
+
+ final Map<String, dynamic> result = <String, dynamic>{};
+ void addResult(dynamic data, String suffix) {
+ assert(data is Map<String, dynamic>);
+ const List<String> metricKeys = <String>[
+ 'janky_count',
+ 'average_abs_jerk',
+ 'dropped_frame_count',
+ ];
+ for (final String key in metricKeys) {
+ result[key+suffix] = data[key];
+ }
+ }
+ addResult(data['resample on with 90Hz input'], '_with_resampler_90Hz');
+ addResult(data['resample on with 59Hz input'], '_with_resampler_59Hz');
+ addResult(data['resample off with 90Hz input'], '_without_resampler_90Hz');
+ addResult(data['resample off with 59Hz input'], '_without_resampler_59Hz');
+
+ return TaskResult.success(
+ result,
+ benchmarkScoreKeys: result.keys.toList(),
+ );
+ });
+ };
+}
+
TaskFunction createFramePolicyIntegrationTest() {
final String testDirectory =
'${flutterDirectory.path}/dev/benchmarks/macrobenchmarks';
diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml
index 730849d..99946af 100644
--- a/dev/devicelab/manifest.yaml
+++ b/dev/devicelab/manifest.yaml
@@ -114,6 +114,14 @@
# Android on-device tests
+ complex_layout_android__scroll_smoothness:
+ description: >
+ Measures the smoothness of scrolling of the Complex Layout sample app on
+ Android.
+ stage: devicelab
+ required_agent_capabilities: ["linux/android"]
+ flaky: true
+
complex_layout_scroll_perf__timeline_summary:
description: >
Measures the runtime performance of the Complex Layout sample app on
diff --git a/packages/flutter_test/lib/src/binding.dart b/packages/flutter_test/lib/src/binding.dart
index fe65b2d..dad55ba 100644
--- a/packages/flutter_test/lib/src/binding.dart
+++ b/packages/flutter_test/lib/src/binding.dart
@@ -1504,7 +1504,6 @@
renderView._pointers[event.pointer].decay = _kPointerDecay;
_handleViewNeedsPaint();
} else if (event.down) {
- assert(event is PointerDownEvent);
renderView._pointers[event.pointer] = _LiveTestPointerRecord(
event.pointer,
event.position,
