[devicelab] Delete run.dart
diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart
deleted file mode 100644
index 26a9f26..0000000
--- a/dev/devicelab/bin/run.dart
+++ /dev/null
@@ -1,379 +0,0 @@
-// Copyright 2014 The Flutter Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'dart:convert';
-import 'dart:io';
-
-import 'package:args/args.dart';
-import 'package:flutter_devicelab/framework/ab.dart';
-import 'package:flutter_devicelab/framework/manifest.dart';
-import 'package:flutter_devicelab/framework/runner.dart';
-import 'package:flutter_devicelab/framework/task_result.dart';
-import 'package:flutter_devicelab/framework/utils.dart';
-import 'package:path/path.dart' as path;
-
-ArgResults args;
-
-List<String> _taskNames = <String>[];
-
-/// The device-id to run test on.
-String deviceId;
-
-/// The git branch being tested on.
-String gitBranch;
-
-/// The build of the local engine to use.
-///
-/// Required for A/B test mode.
-String localEngine;
-
-/// The path to the engine "src/" directory.
-String localEngineSrcPath;
-
-/// Name of the LUCI builder this test is currently running on.
-///
-/// This is only passed on CI runs for Cocoon to be able to uniquely identify
-/// this test run.
-String luciBuilder;
-
-/// Whether to exit on first test failure.
-bool exitOnFirstTestFailure;
-
-/// Path to write test results to.
-String resultsPath;
-
-/// File containing a service account token.
-///
-/// If passed, the test run results will be uploaded to Flutter infrastructure.
-String serviceAccountTokenFile;
-
-/// Suppresses standard output, prints only standard error output.
-bool silent;
-
-/// Runs tasks.
-///
-/// The tasks are chosen depending on the command-line options
-/// (see [_argParser]).
-Future<void> main(List<String> rawArgs) async {
- try {
- args = _argParser.parse(rawArgs);
- } on FormatException catch (error) {
- stderr.writeln('${error.message}\n');
- stderr.writeln('Usage:\n');
- stderr.writeln(_argParser.usage);
- exitCode = 1;
- return;
- }
-
- deviceId = args['device-id'] as String;
- exitOnFirstTestFailure = args['exit'] as bool;
- gitBranch = args['git-branch'] as String;
- localEngine = args['local-engine'] as String;
- localEngineSrcPath = args['local-engine-src-path'] as String;
- luciBuilder = args['luci-builder'] as String;
- resultsPath = args['results-file'] as String;
- serviceAccountTokenFile = args['service-account-token-file'] as String;
- silent = args['silent'] as bool;
-
- if (!args.wasParsed('task')) {
- if (args.wasParsed('stage') || args.wasParsed('all')) {
- addTasks(
- tasks: loadTaskManifest().tasks,
- args: args,
- taskNames: _taskNames,
- );
- }
- }
-
- if (args.wasParsed('list')) {
- for (int i = 0; i < _taskNames.length; i++) {
- print('${(i + 1).toString().padLeft(3)} - ${_taskNames[i]}');
- }
- exitCode = 0;
- return;
- }
-
- if (_taskNames.isEmpty) {
- stderr.writeln('Failed to find tasks to run based on supplied options.');
- exitCode = 1;
- return;
- }
-
- if (args.wasParsed('ab')) {
- await _runABTest();
- } else {
- await runTasks(_taskNames,
- silent: silent,
- localEngine: localEngine,
- localEngineSrcPath: localEngineSrcPath,
- deviceId: deviceId,
- exitOnFirstTestFailure: exitOnFirstTestFailure,
- gitBranch: gitBranch,
- luciBuilder: luciBuilder,
- resultsPath: resultsPath,
- );
- }
-}
-
-Future<void> _runABTest() async {
- final int runsPerTest = int.parse(args['ab'] as String);
-
- if (_taskNames.length > 1) {
- stderr.writeln('When running in A/B test mode exactly one task must be passed but got ${_taskNames.join(', ')}.\n');
- stderr.writeln(_argParser.usage);
- exit(1);
- }
-
- if (!args.wasParsed('local-engine')) {
- stderr.writeln('When running in A/B test mode --local-engine is required.\n');
- stderr.writeln(_argParser.usage);
- exit(1);
- }
-
- final String taskName = _taskNames.single;
-
- print('$taskName A/B test. Will run $runsPerTest times.');
-
- final ABTest abTest = ABTest(localEngine, taskName);
- for (int i = 1; i <= runsPerTest; i++) {
- section('Run #$i');
-
- print('Running with the default engine (A)');
- final TaskResult defaultEngineResult = await runTask(
- taskName,
- silent: silent,
- deviceId: deviceId,
- );
-
- print('Default engine result:');
- print(const JsonEncoder.withIndent(' ').convert(defaultEngineResult));
-
- if (!defaultEngineResult.succeeded) {
- stderr.writeln('Task failed on the default engine.');
- exit(1);
- }
-
- abTest.addAResult(defaultEngineResult);
-
- print('Running with the local engine (B)');
- final TaskResult localEngineResult = await runTask(
- taskName,
- silent: silent,
- localEngine: localEngine,
- localEngineSrcPath: localEngineSrcPath,
- deviceId: deviceId,
- );
-
- print('Task localEngineResult:');
- print(const JsonEncoder.withIndent(' ').convert(localEngineResult));
-
- if (!localEngineResult.succeeded) {
- stderr.writeln('Task failed on the local engine.');
- exit(1);
- }
-
- abTest.addBResult(localEngineResult);
-
- if (!silent && i < runsPerTest) {
- section('A/B results so far');
- print(abTest.printSummary());
- }
- }
- abTest.finalize();
-
- final File jsonFile = _uniqueFile(args['ab-result-file'] as String ?? 'ABresults#.json');
- jsonFile.writeAsString(const JsonEncoder.withIndent(' ').convert(abTest.jsonMap));
-
- if (!silent) {
- section('Raw results');
- print(abTest.rawResults());
- }
-
- section('Final A/B results');
- print(abTest.printSummary());
-
- print('');
- print('Results saved to ${jsonFile.path}');
-}
-
-File _uniqueFile(String filenameTemplate) {
- final List<String> parts = filenameTemplate.split('#');
- if (parts.length != 2) {
- return File(filenameTemplate);
- }
- File file = File(parts[0] + parts[1]);
- int i = 1;
- while (file.existsSync()) {
- file = File(parts[0] + i.toString() + parts[1]);
- i++;
- }
- return file;
-}
-
-void addTasks({
- List<ManifestTask> tasks,
- ArgResults args,
- List<String> taskNames,
-}) {
- if (args.wasParsed('continue-from')) {
- final int index = tasks.indexWhere((ManifestTask task) => task.name == args['continue-from']);
- if (index == -1) {
- throw Exception('Invalid task name "${args['continue-from']}"');
- }
- tasks.removeRange(0, index);
- }
- // Only start skipping if user specified a task to continue from
- final String stage = args['stage'] as String;
- for (final ManifestTask task in tasks) {
- final bool isQualifyingStage = stage == null || task.stage == stage;
- final bool isQualifyingHost = !(args['match-host-platform'] as bool) || task.isSupportedByHost();
- if (isQualifyingHost && isQualifyingStage) {
- taskNames.add(task.name);
- }
- }
-}
-
-/// Command-line options for the `run.dart` command.
-final ArgParser _argParser = ArgParser()
- ..addMultiOption(
- 'task',
- abbr: 't',
- splitCommas: true,
- help: 'Either:\n'
- ' - the name of a task defined in manifest.yaml.\n'
- ' Example: complex_layout__start_up.\n'
- ' - the path to a Dart file corresponding to a task,\n'
- ' which resides in bin/tasks.\n'
- ' Example: bin/tasks/complex_layout__start_up.dart.\n'
- '\n'
- 'This option may be repeated to specify multiple tasks.',
- callback: (List<String> value) {
- for (final String nameOrPath in value) {
- final List<String> fragments = path.split(nameOrPath);
- final bool isDartFile = fragments.last.endsWith('.dart');
-
- if (fragments.length == 1 && !isDartFile) {
- // Not a path
- _taskNames.add(nameOrPath);
- } else if (!isDartFile || !path.equals(path.dirname(nameOrPath), path.join('bin', 'tasks'))) {
- // Unsupported executable location
- throw FormatException('Invalid value for option -t (--task): $nameOrPath');
- } else {
- _taskNames.add(path.withoutExtension(fragments.last));
- }
- }
- },
- )
- ..addOption(
- 'device-id',
- abbr: 'd',
- help: 'Target device id (prefixes are allowed, names are not supported).\n'
- 'The option will be ignored if the test target does not run on a\n'
- 'mobile device. This still respects the device operating system\n'
- 'settings in the test case, and will results in error if no device\n'
- 'with given ID/ID prefix is found.',
- )
- ..addOption(
- 'ab',
- help: 'Runs an A/B test comparing the default engine with the local\n'
- 'engine build for one task. This option does not support running\n'
- 'multiple tasks. The value is the number of times to run the task.\n'
- 'The task is expected to be a benchmark that reports score keys.\n'
- 'The A/B test collects the metrics collected by the test and\n'
- 'produces a report containing averages, noise, and the speed-up\n'
- 'between the two engines. --local-engine is required when running\n'
- 'an A/B test.',
- callback: (String value) {
- if (value != null && int.tryParse(value) == null) {
- throw ArgParserException('Option --ab must be a number, but was "$value".');
- }
- },
- )
- ..addOption(
- 'ab-result-file',
- help: 'The filename in which to place the json encoded results of an A/B test.\n'
- 'The filename may contain a single # character to be replaced by a sequence\n'
- 'number if the name already exists.',
- )
- ..addFlag(
- 'all',
- abbr: 'a',
- help: 'Runs all tasks defined in manifest.yaml in alphabetical order.',
- )
- ..addOption(
- 'continue-from',
- abbr: 'c',
- help: 'With --all or --stage, continue from the given test.',
- )
- ..addFlag(
- 'exit',
- defaultsTo: true,
- help: 'Exit on the first test failure.',
- )
- ..addOption(
- 'git-branch',
- help: '[Flutter infrastructure] Git branch of the current commit. LUCI\n'
- 'checkouts run in detached HEAD state, so the branch must be passed.',
- )
- ..addOption(
- 'local-engine',
- help: 'Name of a build output within the engine out directory, if you\n'
- 'are building Flutter locally. Use this to select a specific\n'
- 'version of the engine if you have built multiple engine targets.\n'
- 'This path is relative to --local-engine-src-path/out. This option\n'
- 'is required when running an A/B test (see the --ab option).',
- )
- ..addFlag(
- 'list',
- abbr: 'l',
- help: "Don't actually run the tasks, but list out the tasks that would\n"
- 'have been run, in the order they would have run.',
- )
- ..addOption(
- 'local-engine-src-path',
- help: 'Path to your engine src directory, if you are building Flutter\n'
- 'locally. Defaults to \$FLUTTER_ENGINE if set, or tries to guess at\n'
- 'the location based on the value of the --flutter-root option.',
- )
- ..addOption('luci-builder', help: '[Flutter infrastructure] Name of the LUCI builder being run on.')
- ..addFlag(
- 'match-host-platform',
- defaultsTo: true,
- help: 'Only run tests that match the host platform (e.g. do not run a\n'
- 'test with a `required_agent_capabilities` value of "mac/android"\n'
- 'on a windows host). Each test publishes its '
- '`required_agent_capabilities`\nin the `manifest.yaml` file.',
- )
- ..addOption(
- 'results-file',
- help: '[Flutter infrastructure] File path for test results. If passed with\n'
- 'task, will write test results to the file.'
- )
- ..addOption(
- 'service-account-token-file',
- help: '[Flutter infrastructure] Authentication for uploading results.',
- )
- ..addOption(
- 'stage',
- abbr: 's',
- help: 'Name of the stage. Runs all tasks for that stage. The tasks and\n'
- 'their stages are read from manifest.yaml.',
- )
- ..addFlag(
- 'silent',
- negatable: true,
- defaultsTo: false,
- )
- ..addMultiOption(
- 'test',
- hide: true,
- splitCommas: true,
- callback: (List<String> value) {
- if (value.isNotEmpty) {
- throw const FormatException(
- 'Invalid option --test. Did you mean --task (-t)?',
- );
- }
- },
- );
