| #!/usr/bin/python -u |
| # Copyright 2018 The Dart project authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| # This script automates the Dart SDK roll steps, including: |
| # - Updating the Dart revision in DEPS |
| # - Updating the Dart dependencies in DEPS |
| # - Syncing dependencies with 'gclient sync' |
| # - Generating GN files for relevant engine configurations |
| # - Building relevant engine configurations |
| # - Running tests in 'example/flutter_gallery' and 'packages/flutter' |
| # - Launching flutter_gallery in release and debug mode |
| # - Running license.sh and updating license files in |
| # 'flutter/ci/licenses_golden' |
| # - Generating a commit with relevant Dart SDK commit logs (optional) |
| # |
| # The following environment variables can be set instead of being passed as |
| # arguments: |
| # - FLUTTER_HOME: the absolute path to the 'flutter' directory |
| # - ENGINE_HOME: the absolute path to the 'engine/src' directory |
| # - DART_SDK_HOME: the absolute path to the root of a Dart SDK project |
| |
| from dart_roll_utils import * |
| import argparse |
| import atexit |
| import datetime |
| import fileinput |
| import os |
| import platform |
| import signal |
| import shutil |
| import subprocess |
| import sys |
| |
| DART_REVISION_ENTRY = 'dart_revision' |
| FLUTTER = '{}/bin/flutter'.format(flutter_home()) |
| FLUTTER_DOCTOR = [FLUTTER, 'doctor'] |
| FLUTTER_RUN = [FLUTTER, 'run'] |
| FLUTTER_TEST = [FLUTTER, 'test'] |
| |
| MAX_GCLIENT_RETRIES = 3 |
| |
| # Returned when licenses do not require updating. |
| LICENSE_SCRIPT_OKAY = 0 |
| # Returned when licenses require updating. |
| LICENSE_SCRIPT_UPDATES = 1 |
| # Returned when either 'pub' or 'dart' isn't in the path. |
| LICENSE_SCRIPT_EXIT_ERROR = 127 |
| |
| CURRENT_SUBPROCESS = None |
| |
| def update_dart_revision(dart_revision): |
| original_revision = '' |
| print_status('Updating Dart revision to {}'.format(dart_revision)) |
| content = get_deps() |
| for idx, line in enumerate(content): |
| if DART_REVISION_ENTRY in line: |
| original_revision = line.strip().split(' ')[1][1:-2] |
| if not is_ancestor_commit(original_revision, |
| dart_revision, |
| dart_sdk_home()): |
| print_error('Dart revision {} is older than existing revision, {}.' + |
| ' Aborting roll.'.format(dart_revision, original_revision)) |
| sys.exit(ERROR_OLD_COMMIT_PROVIDED) |
| |
| content[idx] = " 'dart_revision': '" + dart_revision + "',\n" |
| break |
| write_deps(content) |
| return original_revision |
| |
| def run_process(args, cwd=None, stdout=None): |
| global CURRENT_SUBPROCESS |
| CURRENT_SUBPROCESS = subprocess.Popen(args, cwd=cwd, stdout=stdout) |
| exit_code = CURRENT_SUBPROCESS.wait() |
| CURRENT_SUBPROCESS = None |
| return exit_code |
| |
| def gclient_sync(): |
| global CURRENT_SUBPROCESS |
| exit_code = None |
| num_retries = 0 |
| |
| while ((exit_code != 0) and not (num_retries >= MAX_GCLIENT_RETRIES)): |
| print_status('Running gclient sync (Attempt {}/{})' |
| .format(num_retries + 1, MAX_GCLIENT_RETRIES)) |
| exit_code = run_process(['gclient', 'sync', '--delete_unversioned_trees'], |
| cwd=engine_home()) |
| if exit_code != 0: |
| num_retries += 1 |
| if num_retries == MAX_GCLIENT_RETRIES: |
| print_error('Max number of gclient sync retries attempted. Aborting roll.') |
| sys.exit(ERROR_GCLIENT_SYNC_FAILED) |
| |
| |
| def get_deps(): |
| with open(flutter_deps_path(), 'r') as f: |
| content = f.readlines() |
| return content |
| |
| |
| def update_deps(): |
| print_status('Updating Dart dependencies') |
| run_process([update_dart_deps_path()], cwd=engine_home()) |
| |
| |
| def write_deps(newdeps): |
| with open(flutter_deps_path(), 'w') as f: |
| f.write(''.join(newdeps)) |
| |
| |
| def run_gn(): |
| print_status('Generating build files') |
| common = [os.path.join('flutter', 'tools', 'gn'), '--goma', '--full-dart-sdk'] |
| debug = ['--runtime-mode=debug'] |
| profile = ['--runtime-mode=profile'] |
| release = ['--runtime-mode=release'] |
| runtime_modes = [debug, profile, release] |
| unopt = ['--unoptimized'] |
| android = ['--android'] |
| fuchsia = ['--fuchsia'] |
| |
| for mode in runtime_modes: |
| if set(mode) != set(release): |
| run_process(common + android + unopt + mode, cwd=engine_home()) |
| run_process(common + android + mode, cwd=engine_home()) |
| host = common[:] |
| if set(mode) == set(debug): |
| host += unopt |
| run_process(host + mode, cwd=engine_home()) |
| run_process(common + fuchsia + debug + unopt, cwd=engine_home()) |
| |
| |
| def build(): |
| print_status('Building Flutter engine') |
| command = ['ninja', '-j1000'] |
| configs = [ |
| 'host_debug_unopt', |
| 'host_release', |
| 'host_profile', |
| 'android_debug_unopt', |
| 'android_profile', |
| 'android_release', |
| 'fuchsia_debug_unopt_x64', |
| ] |
| |
| build_dir = 'out' |
| if platform.system() == 'Darwin': |
| build_dir = 'xcodebuild' |
| |
| for config in configs: |
| error_code = run_process(command + ['-C', os.path.join(build_dir, config)], |
| cwd=engine_home()) |
| if error_code != 0: |
| print_error('Build failure for configuration "' + |
| config + |
| '". Aborting roll.') |
| sys.exit(ERROR_BUILD_FAILED) |
| |
| |
| def run_flutter_doctor(): |
| print_status('Running flutter doctor') |
| engine_src_path = '--local-engine-src-path={}'.format(engine_home()) |
| result = run_process(FLUTTER_DOCTOR + ['--local-engine=host_debug_unopt', |
| engine_src_path], |
| cwd=package_flutter_path()) |
| if result != 0: |
| print_error('flutter doctor failed. Aborting roll.') |
| sys.exit(ERROR_FLUTTER_DOCTOR_FAILED) |
| |
| |
| def run_tests(): |
| print_status('Running tests in packages/flutter') |
| engine_src_path = '--local-engine-src-path={}'.format(engine_home()) |
| result = run_process(FLUTTER_TEST + ['--local-engine=host_debug_unopt', |
| engine_src_path], |
| cwd=package_flutter_path()) |
| if result != 0: |
| print_error('package/flutter tests failed. Aborting roll.') |
| sys.exit(ERROR_PKG_FLUTTER_FAILED) |
| |
| print_status('Running tests in examples/flutter_gallery') |
| result = run_process(FLUTTER_TEST + ['--local-engine=host_debug_unopt', |
| engine_src_path, |
| '--disable-service-auth-codes'], |
| cwd=flutter_gallery_path()); |
| if result != 0: |
| print_error('flutter_gallery tests failed. Aborting roll.') |
| sys.exit(ERROR_FLUTTER_GALLERY_FAILED) |
| |
| |
| def run_hot_reload_configurations(): |
| print_status('Running flutter gallery release') |
| engine_src_path = '--local-engine-src-path={}'.format(engine_home()) |
| run_process(FLUTTER_RUN + ['--release', |
| '--local-engine=android_release', |
| engine_src_path], |
| cwd=flutter_gallery_path()) |
| print_status('Running flutter gallery debug') |
| run_process(FLUTTER_RUN + ['--local-engine=android_debug_unopt', |
| engine_src_path], |
| cwd=flutter_gallery_path()) |
| |
| |
| def update_licenses(): |
| print_status('Updating Flutter licenses') |
| result = run_process([engine_license_script_path()], cwd=engine_home()) |
| if result == LICENSE_SCRIPT_EXIT_ERROR: |
| print_error('License script failed to run. Is the Dart SDK (specifically' + |
| ' dart and pub) in your path? Aborting roll.') |
| sys.exit(ERROR_LICENSE_SCRIPT_FAILED) |
| elif (result != LICENSE_SCRIPT_OKAY) and (result != LICENSE_SCRIPT_UPDATES): |
| print_error('Unknown license script error: {}. Aborting roll.' |
| .format(result)) |
| sys.exit(ERROR_LICENSE_SCRIPT_FAILED) |
| |
| # Ignore 'licenses_skia' as they shouldn't change during a Dart SDK roll. |
| src_files = ['licenses_flutter', 'licenses_third_party', 'tool_signature'] |
| for f in src_files: |
| path = os.path.join(license_script_output_path(), f) |
| if os.path.isfile(path): |
| shutil.copy(path, engine_golden_licenses_path()) |
| run_process(['pub', 'get'], cwd=engine_license_script_package_path()) |
| gclient_sync() |
| |
| # Update the LICENSE file. |
| with open(sky_license_file_path(), 'w') as sky_license: |
| run_process(['dart', os.path.join('lib', 'main.dart'), |
| '--release', '--src', engine_home(), |
| '--out', engine_license_script_output_path()], |
| cwd=engine_license_script_package_path(), |
| stdout=sky_license) |
| |
| |
| def get_commit_range(start, finish): |
| range_str = '{}..{}'.format(start, finish) |
| command = ['git', 'log', '--oneline', range_str] |
| orig_dir = os.getcwd() |
| os.chdir(dart_sdk_home()) |
| result = subprocess.check_output(command).decode('utf-8') |
| result = '\n'.join(['dart-lang/sdk@' + l for l in result.splitlines()]) |
| os.chdir(orig_dir) |
| return result |
| |
| |
| def get_short_rev(rev): |
| command = ['git', 'rev-parse', '--short', rev] |
| orig_dir = os.getcwd() |
| os.chdir(dart_sdk_home()) |
| result = subprocess.check_output(command).decode('utf-8').rstrip() |
| os.chdir(orig_dir) |
| return result |
| |
| |
| def git_commit(original_revision, updated_revision): |
| print_status('Committing Dart SDK roll') |
| current_date = datetime.date.today() |
| sdk_log = get_commit_range(original_revision, updated_revision) |
| num_commits = len(sdk_log.splitlines()) |
| commit_msg = ('Roll src/third_party/dart {}..{} ({} commits)' |
| .format(get_short_rev(original_revision), |
| get_short_rev(updated_revision), num_commits)) |
| commit_msg += '\n\n' + sdk_log |
| commit_cmd = ['git', 'commit', '-a', '-m', commit_msg] |
| run_process(commit_cmd, cwd=engine_flutter_path()) |
| |
| |
| def update_roots(args): |
| if args.flutter_home: |
| set_flutter_home(args.flutter_home) |
| |
| if args.engine_home: |
| set_engine_home(args.engine_home) |
| |
| if args.dart_sdk_home: |
| set_dart_sdk_home(args.dart_sdk_home) |
| |
| if flutter_home() == '': |
| print_error('Either "--flutter-home" must be provided or FLUTTER_HOME must' + |
| ' be set. Aborting roll.') |
| sys.exit(ERROR_MISSING_ROOTS) |
| |
| if engine_home() == '': |
| print_error('Either "--engine-home" must be provided or ENGINE_HOME must' + |
| ' be set. Aborting roll.') |
| sys.exit(ERROR_MISSING_ROOTS) |
| |
| if dart_sdk_home() == '': |
| print_error('Either "--dart-sdk-home" must be provided or DART_SDK_HOME ' + |
| 'must be set. Aborting roll.') |
| sys.exit(ERROR_MISSING_ROOTS) |
| |
| |
| def sys_exit(signal, frame): |
| sys.exit() |
| |
| def cleanup_children(): |
| if CURRENT_SUBPROCESS != None: |
| CURRENT_SUBPROCESS.terminate() |
| |
| def main(): |
| parser = argparse.ArgumentParser(description='Automate most Dart SDK roll tasks.') |
| parser.add_argument('--dart-sdk-home', help='Path to the Dart SDK ' + |
| 'repository. Overrides DART_SDK_HOME environment variable') |
| parser.add_argument('dart_sdk_revision', help='Target Dart SDK revision') |
| parser.add_argument('--create-commit', action='store_true', |
| help='Create the engine commit with Dart SDK commit log') |
| parser.add_argument('--engine-home', help='Path to the Flutter engine ' + |
| 'repository. Overrides ENGINE_HOME environment variable') |
| parser.add_argument('--flutter-home', help='Path to the Flutter framework ' + |
| 'repository. Overrides FLUTTER_HOME environment variable') |
| parser.add_argument('--no-build', action='store_true', |
| help='Skip rebuilding the Flutter engine') |
| parser.add_argument('--no-hot-reload', action='store_true', |
| help="Skip hot reload testing") |
| parser.add_argument('--no-test', action='store_true', |
| help='Skip running host tests for package/flutter and ' + |
| 'flutter_gallery') |
| parser.add_argument('--no-update-deps', action='store_true', |
| help='Skip updating DEPS file') |
| parser.add_argument('--no-update-licenses', action='store_true', |
| help='Skip updating licenses') |
| |
| args = parser.parse_args() |
| |
| atexit.register(cleanup_children) |
| signal.signal(signal.SIGTERM, sys_exit) |
| |
| original_revision = None |
| updated_revision = args.dart_sdk_revision |
| |
| # Disable buffering of log output |
| os.environ["PYTHONUNBUFFERED"] = "1" |
| |
| update_roots(args) |
| |
| print_status('Starting Dart SDK roll') |
| if not args.no_update_deps: |
| original_revision = update_dart_revision(updated_revision) |
| gclient_sync() |
| update_deps() |
| gclient_sync() |
| if not args.no_build: |
| run_gn() |
| build() |
| if ((not args.no_test) or (not args.no_hot_reload)): |
| run_flutter_doctor() |
| if not args.no_test: |
| run_tests() |
| if not args.no_hot_reload: |
| run_hot_reload_configurations() |
| if not args.no_update_licenses: |
| update_licenses() |
| if args.create_commit: |
| if original_revision == None: |
| print_warning('"original_revision" not specified. Skipping commit.') |
| print_warning('This happens when the "--no_update_deps" argument is ' + |
| 'provided') |
| else: |
| git_commit(original_revision, updated_revision) |
| print_status('Dart SDK roll complete!') |
| |
| |
| if __name__ == '__main__': |
| main() |