add --exit and --match-host-platform defaults to devicelab runner (#37832)
diff --git a/dev/devicelab/README.md b/dev/devicelab/README.md
index 882726d..c3231f5 100644
--- a/dev/devicelab/README.md
+++ b/dev/devicelab/README.md
@@ -122,6 +122,9 @@
../../bin/cache/dart-sdk/bin/dart bin/run.dart -a
```
+This defaults to only running tests supported by your host device's platform
+(`--match-host-platform`) and exiting after the first failure (`--exit`).
+
## Running specific tests
To run a test, use option `-t` (`--task`):
diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart
index 2d74927..b9f70e6 100644
--- a/dev/devicelab/bin/run.dart
+++ b/dev/devicelab/bin/run.dart
@@ -32,21 +32,23 @@
}
if (!args.wasParsed('task')) {
- if (args.wasParsed('stage')) {
- final String stageName = args['stage'];
- final List<ManifestTask> tasks = loadTaskManifest().tasks;
- for (ManifestTask task in tasks) {
- if (task.stage == stageName)
- _taskNames.add(task.name);
- }
- } else if (args.wasParsed('all')) {
- final List<ManifestTask> tasks = loadTaskManifest().tasks;
- for (ManifestTask task in tasks) {
- _taskNames.add(task.name);
- }
+ if (args.wasParsed('stage') || args.wasParsed('all')) {
+ addTasks(
+ tasks: loadTaskManifest().tasks,
+ args: args,
+ taskNames: _taskNames,
+ );
}
}
+ if (args.wasParsed('list')) {
+ for (int i = 0; i < _taskNames.length; i++) {
+ print('${(i + 1).toString().padLeft(3)} - ${_taskNames[i]}');
+ }
+ exitCode = 0;
+ return;
+ }
+
if (_taskNames.isEmpty) {
stderr.writeln('Failed to find tasks to run based on supplied options.');
exitCode = 1;
@@ -66,12 +68,39 @@
localEngineSrcPath: localEngineSrcPath,
);
- if (!result['success'])
- exitCode = 1;
-
print('Task result:');
print(const JsonEncoder.withIndent(' ').convert(result));
section('Finished task "$taskName"');
+
+ if (!result['success']) {
+ exitCode = 1;
+ if (args['exit']) {
+ return;
+ }
+ }
+ }
+}
+
+void addTasks({
+ List<ManifestTask> tasks,
+ ArgResults args,
+ List<String> taskNames,
+}) {
+ if (args.wasParsed('continue-from')) {
+ final int index = tasks.indexWhere((ManifestTask task) => task.name == args['continue-from']);
+ if (index == -1) {
+ throw Exception('Invalid task name "${args['continue-from']}"');
+ }
+ tasks.removeRange(0, index);
+ }
+ // Only start skipping if user specified a task to continue from
+ final String stage = args['stage'];
+ for (ManifestTask task in tasks) {
+ final bool isQualifyingStage = stage == null || task.stage == stage;
+ final bool isQualifyingHost = !args['match-host-platform'] || task.isSupportedByHost();
+ if (isQualifyingHost && isQualifyingStage) {
+ taskNames.add(task.name);
+ }
}
}
@@ -103,16 +132,58 @@
}
},
)
- ..addOption(
- 'stage',
- abbr: 's',
- help: 'Name of the stage. Runs all tasks for that stage. '
- 'The tasks and their stages are read from manifest.yaml.',
- )
..addFlag(
'all',
abbr: 'a',
- help: 'Runs all tasks defined in manifest.yaml.',
+ help: 'Runs all tasks defined in manifest.yaml in alphabetical order.',
+ )
+ ..addOption(
+ 'continue-from',
+ abbr: 'c',
+ help: 'With --all or --stage, continue from the given test.',
+ )
+ ..addFlag(
+ 'exit',
+ defaultsTo: true,
+ help: 'Exit on the first test failure.',
+ )
+ ..addOption(
+ 'local-engine',
+ help: 'Name of a build output within the engine out directory, if you\n'
+ 'are building Flutter locally. Use this to select a specific\n'
+ 'version of the engine if you have built multiple engine targets.\n'
+ 'This path is relative to --local-engine-src-path/out.',
+ )
+ ..addFlag(
+ 'list',
+ abbr: 'l',
+ help: 'Don\'t actually run the tasks, but list out the tasks that would\n'
+ 'have been run, in the order they would have run.',
+ )
+ ..addOption(
+ 'local-engine-src-path',
+ help: 'Path to your engine src directory, if you are building Flutter\n'
+ 'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
+ 'the location based on the value of the --flutter-root option.',
+ )
+ ..addFlag(
+ 'match-host-platform',
+ defaultsTo: true,
+ help: 'Only run tests that match the host platform (e.g. do not run a\n'
+ 'test with a `required_agent_capabilities` value of "mac/android"\n'
+ 'on a windows host). Each test publishes its'
+ '`required_agent_capabilities`\nin the `manifest.yaml` file.',
+ )
+ ..addOption(
+ 'stage',
+ abbr: 's',
+ help: 'Name of the stage. Runs all tasks for that stage. The tasks and\n'
+ 'their stages are read from manifest.yaml.',
+ )
+ ..addFlag(
+ 'silent',
+ negatable: true,
+ defaultsTo: false,
)
..addMultiOption(
'test',
@@ -125,24 +196,6 @@
);
}
},
- )
- ..addFlag(
- 'silent',
- negatable: true,
- defaultsTo: false,
- )
- ..addOption(
- 'local-engine',
- help: 'Name of a build output within the engine out directory, if you are '
- 'building Flutter locally. Use this to select a specific version of '
- 'the engine if you have built multiple engine targets. This path is '
- 'relative to --local-engine-src-path/out.',
- )
- ..addOption(
- 'local-engine-src-path',
- help: 'Path to your engine src directory, if you are building Flutter '
- 'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at '
- 'the location based on the value of the --flutter-root option.',
);
bool _listsEqual(List<dynamic> a, List<dynamic> b) {
diff --git a/dev/devicelab/lib/framework/manifest.dart b/dev/devicelab/lib/framework/manifest.dart
index dc0eca3..9dc5a76 100644
--- a/dev/devicelab/lib/framework/manifest.dart
+++ b/dev/devicelab/lib/framework/manifest.dart
@@ -3,10 +3,14 @@
// found in the LICENSE file.
import 'package:meta/meta.dart';
+import 'package:platform/platform.dart';
import 'package:yaml/yaml.dart';
import 'utils.dart';
+Platform get platform => _platform ??= const LocalPlatform();
+Platform _platform;
+
/// Loads manifest data from `manifest.yaml` file or from [yaml], if present.
Manifest loadTaskManifest([ String yaml ]) {
final dynamic manifestYaml = yaml == null
@@ -52,7 +56,7 @@
final String stage;
/// Capabilities required of the build agent to be able to perform this task.
- final List<dynamic> requiredAgentCapabilities;
+ final List<String> requiredAgentCapabilities;
/// Whether this test is flaky.
///
@@ -61,6 +65,20 @@
/// An optional custom timeout specified in minutes.
final int timeoutInMinutes;
+
+ /// Whether the task is supported by the current host platform
+ bool isSupportedByHost() {
+ final Set<String> supportedHosts = Set<String>.from(
+ requiredAgentCapabilities.map<String>(
+ (String str) => str.split('/')[0]
+ )
+ );
+ String hostPlatform = platform.operatingSystem;
+ if (hostPlatform == 'macos') {
+ hostPlatform = 'mac'; // package:platform uses 'macos' while manifest.yaml uses 'mac'
+ }
+ return supportedHosts.contains(hostPlatform);
+ }
}
/// Thrown when the manifest YAML is not valid.
diff --git a/dev/devicelab/lib/tasks/sample_catalog_generator.dart b/dev/devicelab/lib/tasks/sample_catalog_generator.dart
index 376cc67..53b225d 100644
--- a/dev/devicelab/lib/tasks/sample_catalog_generator.dart
+++ b/dev/devicelab/lib/tasks/sample_catalog_generator.dart
@@ -39,7 +39,7 @@
await saveCatalogScreenshots(
directory: dir('${flutterDirectory.path}/examples/catalog/.generated'),
commit: commit,
- token: authorizationToken,
+ token: authorizationToken, // TODO(fujino): workaround auth token for local runs
prefix: isIosDevice ? 'ios_' : '',
);
});
