Better filtering for Android `scenario_app` runner. (#50937)
_🍴 'd from https://github.com/flutter/engine/pull/50933, will rebase
when merged._
Closes https://github.com/flutter/flutter/issues/143458.
A picture is a 1000 words:
This is still noisy, but at least all the output appears to be part of
the execution.
As you recall, the full logs are always available in the
FLUTTER_LOGS_DIR output.
diff --git a/testing/scenario_app/bin/run_android_tests.dart b/testing/scenario_app/bin/run_android_tests.dart
index f0ad827..436cb42 100644
--- a/testing/scenario_app/bin/run_android_tests.dart
+++ b/testing/scenario_app/bin/run_android_tests.dart
@@ -47,48 +47,52 @@
..addOption(
'adb',
help: 'Path to the adb tool',
- defaultsTo: engine != null ? join(
- engine.srcDir.path,
- 'third_party',
- 'android_tools',
- 'sdk',
- 'platform-tools',
- 'adb',
- ) : null,
+ defaultsTo: engine != null
+ ? join(
+ engine.srcDir.path,
+ 'third_party',
+ 'android_tools',
+ 'sdk',
+ 'platform-tools',
+ 'adb',
+ )
+ : null,
)
..addOption(
'ndk-stack',
help: 'Path to the ndk-stack tool',
- defaultsTo: engine != null ? join(
- engine.srcDir.path,
- 'third_party',
- 'android_tools',
- 'ndk',
- 'prebuilt',
- () {
- if (Platform.isLinux) {
- return 'linux-x86_64';
- } else if (Platform.isMacOS) {
- return 'darwin-x86_64';
- } else if (Platform.isWindows) {
- return 'windows-x86_64';
- } else {
- throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}');
- }
- }(),
- 'bin',
- 'ndk-stack',
- ) : null,
+ defaultsTo: engine != null
+ ? join(
+ engine.srcDir.path,
+ 'third_party',
+ 'android_tools',
+ 'ndk',
+ 'prebuilt',
+ () {
+ if (Platform.isLinux) {
+ return 'linux-x86_64';
+ } else if (Platform.isMacOS) {
+ return 'darwin-x86_64';
+ } else if (Platform.isWindows) {
+ return 'windows-x86_64';
+ } else {
+ throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}');
+ }
+ }(),
+ 'bin',
+ 'ndk-stack',
+ )
+ : null,
)
..addOption(
'out-dir',
help: 'Out directory',
- defaultsTo:
- engine?.
- outputs().
- where((Output o) => basename(o.path.path).startsWith('android_')).
- firstOrNull?.
- path.path,
+ defaultsTo: engine
+ ?.outputs()
+ .where((Output o) => basename(o.path.path).startsWith('android_'))
+ .firstOrNull
+ ?.path
+ .path,
)
..addOption(
'smoke-test',
@@ -106,14 +110,16 @@
..addOption(
'output-contents-golden',
help: 'Path to a file that contains the expected filenames of golden files.',
- defaultsTo: engine != null ? join(
- engine.srcDir.path,
- 'flutter',
- 'testing',
- 'scenario_app',
- 'android',
- 'expected_golden_output.txt',
- ) : null,
+ defaultsTo: engine != null
+ ? join(
+ engine.srcDir.path,
+ 'flutter',
+ 'testing',
+ 'scenario_app',
+ 'android',
+ 'expected_golden_output.txt',
+ )
+ : null,
)
..addOption(
'impeller-backend',
@@ -124,8 +130,8 @@
..addOption(
'logs-dir',
help: 'The directory to store the logs and screenshots. Defaults to '
- 'the value of the FLUTTER_LOGS_DIR environment variable, if set, '
- 'otherwise it defaults to a path within out-dir.',
+ 'the value of the FLUTTER_LOGS_DIR environment variable, if set, '
+ 'otherwise it defaults to a path within out-dir.',
defaultsTo: Platform.environment['FLUTTER_LOGS_DIR'],
);
@@ -153,7 +159,10 @@
final String? contentsGolden = results['output-contents-golden'] as String?;
final _ImpellerBackend? impellerBackend = _ImpellerBackend.tryParse(results['impeller-backend'] as String?);
if (enableImpeller && impellerBackend == null) {
- panic(<String>['invalid graphics-backend', results['impeller-backend'] as String? ?? '<null>']);
+ panic(<String>[
+ 'invalid graphics-backend',
+ results['impeller-backend'] as String? ?? '<null>'
+ ]);
}
final Directory logsDir = Directory(results['logs-dir'] as String? ?? join(outDir.path, 'scenario_app', 'logs'));
final String? ndkStack = results['ndk-stack'] as String?;
@@ -215,7 +224,10 @@
const ProcessManager pm = LocalProcessManager();
if (!outDir.existsSync()) {
- panic(<String>['out-dir does not exist: $outDir', 'make sure to build the selected engine variant']);
+ panic(<String>[
+ 'out-dir does not exist: $outDir',
+ 'make sure to build the selected engine variant'
+ ]);
}
if (!adb.existsSync()) {
@@ -236,11 +248,17 @@
log('writing logs and screenshots to ${logsDir.path}');
if (!testApk.existsSync()) {
- panic(<String>['test apk does not exist: ${testApk.path}', 'make sure to build the selected engine variant']);
+ panic(<String>[
+ 'test apk does not exist: ${testApk.path}',
+ 'make sure to build the selected engine variant'
+ ]);
}
if (!appApk.existsSync()) {
- panic(<String>['app apk does not exist: ${appApk.path}', 'make sure to build the selected engine variant']);
+ panic(<String>[
+ 'app apk does not exist: ${appApk.path}',
+ 'make sure to build the selected engine variant'
+ ]);
}
// Start a TCP socket in the host, and forward it to the device that runs the tests.
@@ -248,7 +266,7 @@
// for the screenshots.
// On LUCI, the host uploads the screenshots to Skia Gold.
SkiaGoldClient? skiaGoldClient;
- late ServerSocket server;
+ late ServerSocket server;
final List<Future<void>> pendingComparisons = <Future<void>>[];
await step('Starting server...', () async {
server = await ServerSocket.bind(InternetAddress.anyIPv4, _tcpPort);
@@ -259,7 +277,8 @@
if (verbose) {
stdout.writeln('client connected ${client.remoteAddress.address}:${client.remotePort}');
}
- client.transform(const ScreenshotBlobTransformer()).listen((Screenshot screenshot) {
+ client.transform(const ScreenshotBlobTransformer()).listen(
+ (Screenshot screenshot) {
final String fileName = screenshot.filename;
final Uint8List fileContent = screenshot.fileContent;
if (verbose) {
@@ -277,18 +296,15 @@
}
if (isSkiaGoldClientAvailable) {
final Future<void> comparison = skiaGoldClient!
- .addImg(fileName, goldenFile,
- screenshotSize: screenshot.pixelCount)
- .catchError((dynamic err) {
- panic(<String>['skia gold comparison failed: $err']);
- });
+ .addImg(fileName, goldenFile, screenshotSize: screenshot.pixelCount)
+ .catchError((dynamic err) {
+ panic(<String>['skia gold comparison failed: $err']);
+ });
pendingComparisons.add(comparison);
}
- },
- onError: (dynamic err) {
+ }, onError: (dynamic err) {
panic(<String>['error while receiving bytes: $err']);
- },
- cancelOnError: true);
+ }, cancelOnError: true);
});
});
@@ -311,27 +327,38 @@
final (Future<int> logcatExitCode, Stream<String> logcatOutput) = getProcessStreams(logcatProcess);
logcatProcessExitCode = logcatExitCode;
+ String? filterProcessId;
+
logcatOutput.listen((String line) {
// Always write to the full log.
logcat.writeln(line);
// Conditionally parse and write to stderr.
final AdbLogLine? adbLogLine = AdbLogLine.tryParse(line);
- switch (adbLogLine?.process) {
- case null:
- break;
- case 'ActivityManager':
- // These are mostly noise, i.e. "D ActivityManager: freezing 24632 com.blah".
- if (adbLogLine!.severity == 'D') {
- break;
- }
- // TODO(matanlurey): Figure out why this isn't 'flutter.scenario' or similar.
- // Also, why is there two different names?
- case 'utter.scenario':
- case 'utter.scenarios':
- case 'flutter':
- case 'FlutterJNI':
- log('[adb] $line');
+ if (verbose || adbLogLine == null) {
+ log(line);
+ return;
+ }
+
+ // If we haven't already found a process ID, try to find one.
+ // The process ID will help us filter out logs from other processes.
+ filterProcessId ??= adbLogLine.tryParseProcess();
+
+ // If this is a "verbose" log, possibly skip it.
+ final bool isVerbose = adbLogLine.isVerbose(filterProcessId: filterProcessId);
+ if (isVerbose || filterProcessId == null) {
+ // We've requested verbose output, so print everything.
+ if (verbose) {
+ adbLogLine.printFormatted();
+ }
+ return;
+ }
+
+ // It's a non-verbose log, so print it.
+ adbLogLine.printFormatted();
+ }, onError: (Object? err) {
+ if (verbose) {
+ logWarning('logcat stream error: $err');
}
});
});
@@ -364,10 +391,7 @@
log('using dimensions: ${json.encode(dimensions)}');
skiaGoldClient = SkiaGoldClient(
outDir,
- dimensions: <String, String>{
- 'AndroidAPILevel': connectedDeviceAPILevel,
- 'GraphicsBackend': enableImpeller ? 'impeller-${impellerBackend!.name}' : 'skia',
- },
+ dimensions: dimensions,
);
});
@@ -412,11 +436,9 @@
'am',
'instrument',
'-w',
- if (smokeTestFullPath != null)
- '-e class $smokeTestFullPath',
+ if (smokeTestFullPath != null) '-e class $smokeTestFullPath',
'dev.flutter.scenarios.test/dev.flutter.TestRunner',
- if (enableImpeller)
- '-e enable-impeller',
+ if (enableImpeller) '-e enable-impeller',
if (impellerBackend != null)
'-e impeller-backend ${impellerBackend.name}',
]);
@@ -465,7 +487,8 @@
final int exitCode = await pm.runAndForward(<String>[
adb.path,
'reverse',
- '--remove', 'tcp:3000',
+ '--remove',
+ 'tcp:3000',
]);
if (exitCode != 0) {
panic(<String>['could not unforward port']);
@@ -473,14 +496,16 @@
});
await step('Uninstalling app APK...', () async {
- final int exitCode = await pm.runAndForward(<String>[adb.path, 'uninstall', 'dev.flutter.scenarios']);
+ final int exitCode = await pm.runAndForward(
+ <String>[adb.path, 'uninstall', 'dev.flutter.scenarios']);
if (exitCode != 0) {
panic(<String>['could not uninstall app apk']);
}
});
await step('Uninstalling test APK...', () async {
- final int exitCode = await pm.runAndForward(<String>[adb.path, 'uninstall', 'dev.flutter.scenarios.test']);
+ final int exitCode = await pm.runAndForward(
+ <String>[adb.path, 'uninstall', 'dev.flutter.scenarios.test']);
if (exitCode != 0) {
panic(<String>['could not uninstall app apk']);
}
diff --git a/testing/scenario_app/bin/utils/adb_logcat_filtering.dart b/testing/scenario_app/bin/utils/adb_logcat_filtering.dart
index ca67ab1..9785263 100644
--- a/testing/scenario_app/bin/utils/adb_logcat_filtering.dart
+++ b/testing/scenario_app/bin/utils/adb_logcat_filtering.dart
@@ -1,3 +1,7 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
/// Some notes about filtering `adb logcat` output, especially as a result of
/// running `adb shell` to instrument the app and test scripts, as it's
/// non-trivial and error-prone.
@@ -26,6 +30,8 @@
/// See also: <https://developer.android.com/tools/logcat>.
library;
+import 'logs.dart';
+
/// Represents a line of `adb logcat` output parsed into a structured form.
///
/// For example the line:
@@ -40,14 +46,15 @@
/// with lazy parsing.
extension type const AdbLogLine._(Match _match) {
// RegEx that parses into the following groups:
- // 1. Everything up to the severity (I, W, E, etc.).
- // In other words, any whitespace, numbers, hyphens, colons, and periods.
- // 2. The severity (a single uppercase letter).
- // 3. The name of the process (up to the colon).
- // 4. The message (after the colon).
+ // 1. The time of the log message, such as `02-22 13:54:39.839`.
+ // 2. The process ID.
+ // 3. The thread ID.
+ // 4. The character representing the severity of the log message, such as `I`.
+ // 5. The tag, such as `ActivityManager`.
+ // 6. The actual log message.
//
// This regex is simple versus being more precise. Feel free to improve it.
- static final RegExp _pattern = RegExp(r'([^A-Z]*)([A-Z])\s([^:]*)\:\s(.*)');
+ static final RegExp _pattern = RegExp(r'(\d+-\d+\s[\d|:]+\.\d+)\s+(\d+)\s+(\d+)\s(\w)\s(\S+)\s*:\s*(.*)');
/// Parses the given [adbLogCatLine] into a structured form.
///
@@ -57,15 +64,91 @@
return match == null ? null : AdbLogLine._(match);
}
+ /// Tries to parse the process that was started, if the log line is about it.
+ String? tryParseProcess() {
+ if (name == 'ActivityManager' && message.startsWith('Start proc')) {
+ // Start proc 6840:d
+ final RegExpMatch? match = RegExp(r'Start proc (\d+):').firstMatch(message);
+ return match?.group(1);
+ }
+ return null;
+ }
+
+ /// Returns `true` if the log line is verbose.
+ bool isVerbose({String? filterProcessId}) => !_isRelevant(filterProcessId: filterProcessId);
+ bool _isRelevant({String? filterProcessId}) {
+ // Fatal errors are always useful.
+ if (severity == 'F') {
+ return true;
+ }
+
+ // Debug logs are rarely useful.
+ if (severity == 'D') {
+ return false;
+ }
+
+ // These are "known" noise tags.
+ if (const <String>{
+ 'MonitoringInstr',
+ 'ResourceExtractor',
+ 'THREAD_STATE',
+ 'ziparchive',
+ }.contains(name)) {
+ return false;
+ }
+
+ // These are "known" tags useful for debugging.
+ if (const <String>{
+ 'utter.scenario',
+ 'utter.scenarios',
+ 'TestRunner',
+ }.contains(name)) {
+ return true;
+ }
+
+ // If a process ID is specified, exclude logs _not_ from that process.
+ if (filterProcessId != null && process != filterProcessId) {
+ return false;
+ }
+
+ // And... whatever, include anything with the word "flutter".
+ return name.toLowerCase().contains('flutter') || message.toLowerCase().contains('flutter');
+ }
+
+ /// Logs the line to the console.
+ void printFormatted() {
+ final String formatted = '$time [$severity] $name: $message';
+ if (severity == 'W' || severity == 'E' || severity == 'F') {
+ logWarning(formatted);
+ } else if (name == 'TestRunner') {
+ logImportant(formatted);
+ } else {
+ log(formatted);
+ }
+ }
+
/// The full line of `adb logcat` output.
String get line => _match.group(0)!;
- /// The character representing the severity of the log message, such as `I`.
- String get severity => _match.group(2)!;
+ /// The time of the log message, such as `02-22 13:54:39.839`.
+ String get time => _match.group(1)!;
- /// The process name, such as `ActivityManager`.
- String get process => _match.group(3)!;
+ /// The process ID.
+ String get process => _match.group(2)!;
+
+ /// The thread ID.
+ String get thread => _match.group(3)!;
+
+ /// The character representing the severity of the log message, such as `I`.
+ String get severity => _match.group(4)!;
+
+ /// The tag, such as `ActivityManager`.
+ String get name => _match.group(5)!;
/// The actual log message.
- String get message => _match.group(4)!;
+ String get message => _match.group(6)!;
+
+ String toDebugString() {
+ return 'AdbLogLine(time: $time, process: $process, thread: $thread, severity: $severity, name: $name, message: $message)';
+ }
}
diff --git a/testing/scenario_app/bin/utils/logs.dart b/testing/scenario_app/bin/utils/logs.dart
index b84e812..4d883d3 100644
--- a/testing/scenario_app/bin/utils/logs.dart
+++ b/testing/scenario_app/bin/utils/logs.dart
@@ -7,6 +7,7 @@
bool _supportsAnsi = stdout.supportsAnsiEscapes;
String _green = _supportsAnsi ? '\u001b[1;32m' : '';
String _red = _supportsAnsi ? '\u001b[31m' : '';
+String _yellow = _supportsAnsi ? '\u001b[33m' : '';
String _gray = _supportsAnsi ? '\u001b[90m' : '';
String _reset = _supportsAnsi? '\u001B[0m' : '';
@@ -22,15 +23,27 @@
}
}
+void _logWithColor(String color, String msg) {
+ stdout.writeln('$color$msg$_reset');
+}
+
void log(String msg) {
- stdout.writeln('$_gray$msg$_reset');
+ _logWithColor(_gray, msg);
+}
+
+void logImportant(String msg) {
+ stdout.writeln(msg);
+}
+
+void logWarning(String msg) {
+ _logWithColor(_yellow, msg);
}
final class Panic extends Error {}
Never panic(List<String> messages) {
for (final String message in messages) {
- stderr.writeln('$_red$message$_reset');
+ _logWithColor(_red, message);
}
throw Panic();
}
diff --git a/testing/scenario_app/tool/logcat_reader.dart b/testing/scenario_app/tool/logcat_reader.dart
new file mode 100644
index 0000000..909204a
--- /dev/null
+++ b/testing/scenario_app/tool/logcat_reader.dart
@@ -0,0 +1,43 @@
+// Copyright 2013 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io' as io;
+
+// It's bad to import a file from `bin` into `tool`.
+// However this tool is not very important, so delete it if necessary.
+import '../bin/utils/adb_logcat_filtering.dart';
+
+/// A tiny tool to read saved `adb logcat` output and perform some analysis.
+///
+/// This tool is not meant to be a full-fledged logcat reader. It's just a
+/// simple tool that uses the [AdbLogLine] extension type to parse results of
+/// `adb logcat` and explain what log tag names are most common.
+void main(List<String> args) {
+ if (args case [final String path]) {
+ final List<AdbLogLine> parsed = io.File(path)
+ .readAsLinesSync()
+ .map(AdbLogLine.tryParse)
+ .whereType<AdbLogLine>()
+ // Filter out all debug logs.
+ .where((AdbLogLine line) => line.severity != 'D')
+ .toList();
+
+ final Map<String, int> tagCounts = <String, int>{};
+ for (final AdbLogLine line in parsed) {
+ tagCounts[line.name] = (tagCounts[line.name] ?? 0) + 1;
+ }
+
+ // Print in order of most common to least common.
+ final List<MapEntry<String, int>> sorted = tagCounts.entries.toList()
+ ..sort((MapEntry<String, int> a, MapEntry<String, int> b) => b.value.compareTo(a.value));
+ for (final MapEntry<String, int> entry in sorted) {
+ print("'${entry.key}', // ${entry.value}");
+ }
+
+ return;
+ }
+
+ print('Usage: logcat_reader.dart <path-to-logcat-output>');
+ io.exitCode = 1;
+}
