Fix Android `scenario_app` to actually run and block on CI (#50414)
https://github.com/flutter/engine/pull/50414/commits/20337eaea438e505d4bd215b4e84362fa9935d2d
(now reverted) intentionally re-introduced a bug
(https://github.com/flutter/engine/pull/49938) on a now working test
runner, and verified that (with some fixes in this PR), we can now catch
regressions on CI with SkiaGold:
---
<details>
<summary>ExternalTextureTests_testCanvasSurface.png</summary>
This test shows that what previously was a picture has been reduced down
to a single stretched pixel.
**BEFORE**:
**AFTER**:
</details>
<details>
<summary>ExternalTextureTests_testRotatedMediaSurface_180.png</summary>
Similar to above, but shows _another_ bug (kClamp versus kRepeat)
**BEFORE**:
**AFTER**:
/cc
</details>
---
This PR now makes the `scenario_test` useful and blocking for Android
engine rolls:
- Add support for connecting to SkiaGold by depending on `goldctl` in
the appropriate configs.
- Add some useful local debugging flags for skipping SkiaGold or running
specific tests (not used by default).
- Failing to connect to Skia gold is now a test failure.
diff --git a/.ci.yaml b/.ci.yaml
index 45cbaac..5d9d990 100644
--- a/.ci.yaml
+++ b/.ci.yaml
@@ -54,6 +54,10 @@
recipe: engine_v2/engine_v2
properties:
config_name: linux_android_emulator
+ dependencies: >-
+ [
+ {"dependency": "goldctl", "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"}
+ ]
timeout: 60
runIf:
- .ci.yaml
@@ -71,6 +75,10 @@
recipe: engine_v2/engine_v2
properties:
config_name: linux_android_emulator_api_33
+ dependencies: >-
+ [
+ {"dependency": "goldctl", "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"}
+ ]
timeout: 60
runIf:
- .ci.yaml
diff --git a/ci/builders/linux_android_emulator.json b/ci/builders/linux_android_emulator.json
index 4ef64af..643eb4d 100644
--- a/ci/builders/linux_android_emulator.json
+++ b/ci/builders/linux_android_emulator.json
@@ -17,6 +17,12 @@
"--rbe",
"--no-goma"
],
+ "dependencies": [
+ {
+ "dependency": "goldctl",
+ "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"
+ }
+ ],
"name": "android_debug_x64",
"ninja": {
"config": "android_debug_x64",
diff --git a/ci/builders/linux_android_emulator_api_33.json b/ci/builders/linux_android_emulator_api_33.json
index 3e518f1..2457782 100644
--- a/ci/builders/linux_android_emulator_api_33.json
+++ b/ci/builders/linux_android_emulator_api_33.json
@@ -17,6 +17,12 @@
"--rbe",
"--no-goma"
],
+ "dependencies": [
+ {
+ "dependency": "goldctl",
+ "version": "git_revision:720a542f6fe4f92922c3b8f0fdcc4d2ac6bb83cd"
+ }
+ ],
"name": "android_debug_x64",
"ninja": {
"config": "android_debug_x64",
diff --git a/testing/scenario_app/bin/android_integration_tests.dart b/testing/scenario_app/bin/android_integration_tests.dart
index 9517ee3..07b31d0 100644
--- a/testing/scenario_app/bin/android_integration_tests.dart
+++ b/testing/scenario_app/bin/android_integration_tests.dart
@@ -21,18 +21,23 @@
final ArgParser parser = ArgParser()
..addOption(
'adb',
- help: 'absolute path to the adb tool',
+ help: 'Absolute path to the adb tool',
mandatory: true,
)
..addOption(
'out-dir',
- help: 'out directory',
+ help: 'Out directory',
mandatory: true,
)
- ..addFlag(
+ ..addOption(
'smoke-test',
help: 'runs a single test to verify the setup',
- negatable: false,
+ valueHelp: 'The class to execute, defaults to dev.flutter.scenarios.EngineLaunchE2ETest',
+ )
+ ..addFlag(
+ 'use-skia-gold',
+ help: 'Use Skia Gold to compare screenshots.',
+ defaultsTo: isLuciEnv,
);
runZonedGuarded(
@@ -40,8 +45,17 @@
final ArgResults results = parser.parse(args);
final Directory outDir = Directory(results['out-dir'] as String);
final File adb = File(results['adb'] as String);
- final bool smokeTest = results['smoke-test'] as bool;
- await _run(outDir: outDir, adb: adb, smokeTest: smokeTest);
+ final bool useSkiaGold = results['use-skia-gold'] as bool;
+ String? smokeTest = results['smoke-test'] as String?;
+ if (results.wasParsed('smoke-test') && smokeTest!.isEmpty) {
+ smokeTest = 'dev.flutter.scenarios.EngineLaunchE2ETest';
+ }
+ await _run(
+ outDir: outDir,
+ adb: adb,
+ smokeTestFullPath: smokeTest,
+ useSkiaGold: useSkiaGold,
+ );
exit(0);
},
(Object error, StackTrace stackTrace) {
@@ -57,7 +71,8 @@
Future<void> _run({
required Directory outDir,
required File adb,
- required bool smokeTest,
+ required String? smokeTestFullPath,
+ required bool useSkiaGold,
}) async {
const ProcessManager pm = LocalProcessManager();
@@ -178,7 +193,11 @@
await skiaGoldClient!.auth();
log('skia gold client is available');
} else {
- log('skia gold client is unavailable');
+ if (useSkiaGold) {
+ panic(<String>['skia gold client is unavailable']);
+ } else {
+ log('skia gold client is unavaialble');
+ }
}
});
@@ -210,8 +229,8 @@
'am',
'instrument',
'-w',
- if (smokeTest)
- '-e class dev.flutter.scenarios.EngineLaunchE2ETest',
+ if (smokeTestFullPath != null)
+ '-e class $smokeTestFullPath',
'dev.flutter.scenarios.test/dev.flutter.TestRunner',
]);
if (exitCode != 0) {
