| #!/usr/bin/python -u |
| # Copyright 2019 The Dart project authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| # |
| # This script automates the steps required to fully roll a recent version of the |
| # Dart SDK into the Flutter engine (and can easily be extended to roll into the |
| # Flutter framework as well). |
| # |
| # The following steps are completed as part of the roll: |
| # - The Dart buildbots are queried to determine which SDK revision should be |
| # used. Only revisions which have finished all VM and Flutter builds will be |
| # considered, and revisions with < 90% of the VM bots green are ignored. See |
| # dart_buildbot_helper.py for more details. |
| # - dart_roll_helper.py is run with the chosen revision. This performs all the |
| # interesting parts of the roll including running tests. If the steps all |
| # complete successfully, dart_roll_helper.py exits with code 0. Other |
| # possible statuses are listed in dart_roll_utils.py, and will cause this |
| # script to exit with a non-zero exit code. dart_roll_helper.py can also be |
| # run manually to perform a Dart SDK roll with specific parameters. |
| # - The commit created by dart_roll_helper.py is pushed to a branch in |
| # flutter/engine and a pull request is created. Once all PR checks are |
| # complete, the PR is either merged or closed and all state created by this |
| # script is cleaned up. |
| # |
| # In order for this script to work, the following environment variables much be |
| # set: |
| # - GITHUB_API_KEY: A GitHub personal access token for the GitHub account to |
| # be used for uploading the SDK roll changes (see |
| # https://github.com/settings/tokens) |
| # - FLUTTER_HOME: the absolute path to the 'flutter' directory |
| # - ENGINE_HOME: the absolute path to the 'engine/src' directory |
| # - DART_SDK_HOME: the absolute path to the root of a Dart SDK project |
| # - ENGINE_FORK: the name of the GitHub fork to use (e.g., bkonyi/engine or |
| # flutter/engine) |
| # |
| # Finally, the following pip commands need to be run: |
| # - `pip install gitpython` for GitPython (git) |
| # - `pip install PyGithub` for PyGithub (github) |
| |
| from dart_buildbot_helper import get_most_recent_green_build |
| from dart_roll_utils import * |
| from git import Repo |
| from github import Github, GithubException |
| import argparse |
| import atexit |
| import datetime |
| import os |
| import signal |
| import shutil |
| import subprocess |
| import sys |
| import time |
| |
| GITHUB_STATUS_FAILURE = 'failure' |
| GITHUB_STATUS_PENDING = 'pending' |
| GITHUB_STATUS_SUCCESS = 'success' |
| |
| PULL_REQUEST_DESCRIPTION = ( |
| 'This is an automated pull request which will automatically merge once ' |
| 'checks pass.' |
| ) |
| |
| FLAG_skip_wait_for_artifacts = False |
| |
| CURRENT_SUBPROCESS = None |
| CURRENT_PR = None |
| GITHUB_ENGINE_REPO = None |
| GITHUB_ENGINE_FORK = None |
| |
| def run_dart_roll_helper(most_recent_commit, extra_args): |
| global CURRENT_SUBPROCESS |
| os.environ["PYTHONUNBUFFERED"] = "1" |
| args = ['python', |
| os.path.join(os.path.dirname(__file__), 'dart_roll_helper.py'), |
| '--no-hot-reload', |
| most_recent_commit] + extra_args |
| CURRENT_SUBPROCESS = subprocess.Popen(args) |
| result = CURRENT_SUBPROCESS.wait() |
| CURRENT_SUBPROCESS = None |
| return result |
| |
| |
| # TODO(bkonyi): uncomment if we decide to roll into the framework. |
| # def get_engine_version_path(flutter_repo_path): |
| # return os.path.join(flutter_repo_path, |
| # 'bin', |
| # 'internal', |
| # 'engine.version') |
| # |
| # |
| # def get_current_engine_version(flutter_repo_path): |
| # with open(get_engine_version_path(flutter_repo_path), 'r') as f: |
| # return f.readline().strip() |
| # |
| # |
| # def update_engine_version(flutter_repo_path, sha): |
| # with open(get_engine_version_path(flutter_repo_path), 'w') as f: |
| # f.write(sha) |
| # |
| # |
| # def run_engine_roll_helper(engine_local_repo, |
| # flutter_repo_path, |
| # flutter_local_repo, |
| # flutter_github_repo): |
| # clean_and_update_repo(engine_local_repo) |
| # engine_commits = list(engine_local_repo.iter_commits())[:2] |
| # pre_roll_commit = engine_commits[1].hexsha |
| # roll_commit = engine_commits[0].hexsha |
| # current_engine_version = get_current_engine_version(flutter_repo_path) |
| # |
| # # Run `flutter doctor` until artifacts are uploaded to cloud. |
| # wait_for_engine_artifacts(flutter_repo_path, roll_commit) |
| # |
| # # Update the engine repo again to get any changes that may have gone in while |
| # # waiting for the artifacts to build. |
| # clean_and_update_repo(engine_local_repo) |
| # |
| # if not is_ancestor_commit(current_engine_version, |
| # roll_commit, |
| # engine_flutter_path()): |
| # print_status(('Existing revision {} already contains the Dart SDK roll. ' |
| # 'No more work to do!').format(current_engine_version)) |
| # sys.exit(ERROR_ROLL_SUCCESS) |
| # |
| # current_date = datetime.datetime.today().strftime('%Y-%m-%d') |
| # branch_name = 'dart-sdk-roll-{}'.format(current_date) |
| # engine_version_path = get_engine_version_path(flutter_repo_path) |
| # pr_name = 'Dart SDK roll for {}'.format(current_date) |
| # |
| # if pre_roll_commit != current_engine_version: |
| # # Update the engine version to the commit before the Dart SDK roll. |
| # # This ensures that the Dart SDK version bump is the only change in |
| # # the engine roll. |
| # update_engine_version(flutter_repo_path, pre_roll_commit) |
| # create_commit(flutter_local_repo, |
| # branch_name, |
| # 'Roll engine ahead of Dart SDK roll', |
| # [engine_version_path]) |
| # |
| # # Actually update the engine version to include the Dart SDK version bump. |
| # update_engine_version(flutter_repo_path, roll_commit) |
| # create_commit(flutter_local_repo, |
| # branch_name, |
| # 'Roll engine with Dart SDK roll', |
| # [engine_version_path]) |
| # |
| # pull_request = create_pull_request(flutter_github_repo, |
| # flutter_local_repo, |
| # pr_name, |
| # branch_name) |
| # |
| # merge_on_success(flutter_github_repo, pull_request) |
| # |
| # |
| # def wait_for_engine_artifacts(flutter_repo_path, engine_revision): |
| # if FLAG_skip_wait_for_artifacts: |
| # print_warning('Skipping wait for Flutter engine artifacts.') |
| # return |
| # |
| # flutter_tools = os.path.join(flutter_repo_path, 'bin', 'flutter') |
| # cache_path = os.path.join(flutter_repo_path, 'bin', 'cache') |
| # |
| # # Run `flutter doctor` until it can successfully find the engine artifacts |
| # args = [flutter_tools, |
| # 'doctor', |
| # '--check-for-remote-artifacts', |
| # engine_revision] |
| # while True: |
| # result = subprocess.Popen(args, stdout=subprocess.DEVNULL).wait() |
| # if result == 0: |
| # break |
| # time.sleep(15) |
| # |
| # |
| # def create_commit(local_repo, branch, message, files): |
| # local_repo.create_head(branch) |
| # local_repo.git.checkout(branch) |
| # index = local_repo.index |
| # index.add(files) |
| # index.commit(message) |
| |
| |
| def clean_and_update_repo(local_repo): |
| local_repo.git.checkout('.') |
| local_repo.git.clean('-xdf') |
| local_repo.git.checkout('master') |
| local_repo.git.pull() |
| |
| def clean_and_update_forked_repo(local_repo): |
| local_repo.git.checkout('.') |
| local_repo.git.clean('-xdf') |
| local_repo.git.fetch('upstream') |
| local_repo.git.checkout('master') |
| local_repo.git.merge('upstream/master') |
| |
| def clean_build_outputs(): |
| print_status('Cleaning build directory...') |
| args = ['rm', '-rf', |
| os.path.join(ENGINE_HOME, 'out')] |
| CURRENT_SUBPROCESS = subprocess.Popen(args) |
| CURRENT_SUBPROCESS.wait() |
| CURRENT_SUBPROCESS = None |
| |
| def delete_local_branch(local_repo, branch): |
| print_status('Deleting local branch {} in: {}'.format( |
| branch, |
| local_repo.working_tree_dir)) |
| local_repo.git.checkout('master') |
| local_repo.delete_head(branch, '-D') |
| |
| |
| def delete_remote_branch(github_repo, branch): |
| print_status('Deleting remote branch on {}: {}'.format(github_repo.full_name, |
| branch)) |
| github_repo.get_git_ref('heads/{}'.format(branch)).delete() |
| |
| |
| def get_most_recent_commit(local_repo): |
| commits = list(local_repo.iter_commits())[:1] |
| return commits[0] |
| |
| |
| def get_pr_title(local_repo): |
| commit = get_most_recent_commit(local_repo) |
| return commit.message.splitlines()[0].rstrip() |
| |
| |
| def create_pull_request(github_repo, local_repo, title, branch): |
| local_repo.create_head(branch) |
| local_repo.git.checkout(branch) |
| local_repo.git.push('origin', branch) |
| commit = get_most_recent_commit(local_repo) |
| description = PULL_REQUEST_DESCRIPTION + '\n\n' + commit.message |
| try: |
| return github_repo.create_pull(title, description, |
| 'master', '{}:{}'.format('bkonyi', branch)) |
| except GithubException as e: |
| delete_remote_branch(GITHUB_ENGINE_FORK, branch) |
| raise DartAutorollerException(e.data['errors'][0]['message']) |
| finally: |
| print_status('Cleaning up local branch: {}'.format(branch)) |
| delete_local_branch(local_repo, branch) |
| # Remove the commit from the local master branch. |
| local_repo.git.reset('--hard','origin/master') |
| |
| |
| def cleanup_pr(github_repo, pull_request, reason): |
| msg = '{}, abandoning roll.'.format(reason) |
| pull_request.create_issue_comment(msg) |
| pull_request.edit(state='closed') |
| print_error(msg) |
| delete_remote_branch(GITHUB_ENGINE_FORK, pull_request.head.ref) |
| |
| |
| def merge_on_success(github_repo, local_repo, pull_request): |
| sha = pull_request.head.sha |
| commit = github_repo.get_commit(sha=sha) |
| |
| # TODO(bkonyi): Handle case where Flutter tree is red and we're trying to |
| # merge into flutter/flutter. |
| should_merge = wait_for_status(pull_request, commit) |
| if should_merge: |
| if not pull_request.is_merged(): |
| pull_request.create_issue_comment('Checks successful, automatically merging.') |
| merge_status = pull_request.merge(merge_method='squash').merged |
| if not merge_status: |
| print_error('Merge failed! Aborting roll.') |
| sys.exit(1) |
| print_status('Merge was successful!') |
| else: |
| print_status('Manual merge was performed.') |
| else: |
| cleanup_pr(github_repo, pull_request, 'Checks failed') |
| sys.exit(1) |
| delete_remote_branch(GITHUB_ENGINE_FORK, pull_request.head.ref) |
| |
| |
| # TODO(bkonyi): Check to see if the Flutter build is green for flutter/flutter |
| # if we decide to roll the engine into the framework. |
| # def flutter_build_passing(commit): |
| # FLUTTER_BUILD = 'flutter-build' |
| # statuses = commit.get_statuses() |
| # for status in statuses: |
| # if status.context == FLUTTER_BUILD: |
| # return (status.state == GITHUB_STATUS_SUCCESS) |
| # If flutter-build isn't a valid status, the PR checks don't require the |
| # Flutter framework to be green to submit. |
| # return True |
| |
| # Determines if any failures are actual failures from the PR or are just |
| # failures from the engine builders. |
| def is_only_engine_build_failing(commit): |
| BUILD = '-build' |
| LUCI_ENGINE = 'luci-engine' |
| statuses = commit.get_statuses() |
| for status in statuses: |
| if (not ((BUILD in status.context) or |
| (LUCI_ENGINE == status.context)) and |
| (status.state == GITHUB_STATUS_FAILURE)): |
| return False |
| print_status("An engine builder is still failing...") |
| return True |
| |
| |
| def wait_for_status(pull_request, commit): |
| if FLAG_skip_wait_for_artifacts: |
| return True |
| |
| print_status('Sleeping for 120 seconds to allow for Cirrus to start...') |
| |
| # Give Cirrus a chance to start. The GitHub statuses posted by Cirrus go |
| # through some weird states when the PR is created and can be marked as |
| # failing temporarily, causing this check to return False if we don't wait. |
| time.sleep(120) |
| |
| print_status('Starting PR status checks (this may take awhile).') |
| |
| # Ensure all checks pass. |
| while True: |
| # Check to see if the PR was manually merged. |
| if pull_request.is_merged(): |
| break |
| status = commit.get_combined_status().state |
| if status == GITHUB_STATUS_SUCCESS: |
| break |
| # If the only the engine builders are red, keep trying. |
| elif ((status == GITHUB_STATUS_FAILURE) and |
| (not is_only_engine_build_failing(commit))): |
| return False |
| time.sleep(5) |
| |
| # TODO(bkonyi): Re-enable this check if we decide to roll the engine into the |
| # framework. |
| # Once all checks are passing, wait for the Flutter build to be green. |
| # while not flutter_build_passing(commit): |
| # print_status('Waiting for Flutter build to pass...') |
| # time.sleep(60) |
| # print_status('Flutter build passing!') |
| return True |
| |
| |
| def sys_exit(signal, frame): |
| sys.exit() |
| |
| |
| def cleanup_children(): |
| print_error('Roll canceled! Shutting down dart_autoroller.py.') |
| if CURRENT_SUBPROCESS != None: |
| CURRENT_SUBPROCESS.terminate() |
| if CURRENT_PR != None: |
| cleanup_pr(GITHUB_ENGINE_REPO, CURRENT_PR, 'Canceled by roller') |
| |
| |
| def main(): |
| global CURRENT_PR |
| global FLAG_skip_wait_for_artifacts |
| global GITHUB_ENGINE_REPO |
| global GITHUB_ENGINE_FORK |
| |
| parser = argparse.ArgumentParser(description='Dart SDK autoroller for Flutter.') |
| parser.add_argument('--dart-sdk-revision', |
| help='Provide a Dart SDK revision to roll instead of ' |
| 'choosing one automatically') |
| parser.add_argument('--no-update-repos', |
| help='Skip cleaning and updating local repositories', |
| action='store_true') |
| parser.add_argument('--skip-roll', |
| help='Skip running dart_roll_helper.py', |
| action='store_true') |
| parser.add_argument('--skip-tests', |
| help='Skip running Flutter tests', |
| action='store_true') |
| parser.add_argument('--skip-build', |
| help='Skip building all configurations', |
| action='store_true') |
| parser.add_argument('--skip-update-deps', |
| help='Skip updating the Dart SDK dependencies', |
| action='store_true') |
| parser.add_argument('--skip-wait-for-artifacts', |
| help="Don't wait for PR statuses to pass or for engine" + |
| " artifacts to be uploaded to the cloud", |
| action='store_true', default=False) |
| parser.add_argument('--skip-update-licenses', |
| help='Skip updating the licenses for the Flutter engine', |
| action='store_true') |
| parser.add_argument('--skip-pull-request', |
| help="Skip creating a pull request and don't commit", |
| action='store_true') |
| |
| args = parser.parse_args() |
| FLAG_skip_wait_for_artifacts = args.skip_wait_for_artifacts |
| github_api_key = os.getenv('GITHUB_API_KEY') |
| dart_sdk_path = os.getenv('DART_SDK_HOME') |
| flutter_path = os.getenv('FLUTTER_HOME') |
| engine_path = os.getenv('ENGINE_HOME') |
| engine_fork = os.getenv('ENGINE_FORK') |
| local_dart_sdk_repo = Repo(dart_sdk_path) |
| local_flutter_repo = Repo(flutter_path) |
| local_engine_flutter_repo = Repo(os.path.join(engine_path, 'flutter')) |
| assert(not local_dart_sdk_repo.bare) |
| assert(not local_flutter_repo.bare) |
| assert(not local_engine_flutter_repo.bare) |
| |
| github = Github(github_api_key) |
| GITHUB_ENGINE_REPO = github.get_repo('flutter/engine') |
| GITHUB_ENGINE_FORK = github.get_repo(engine_fork) |
| github_flutter_repo = github.get_repo('flutter/flutter') |
| |
| atexit.register(cleanup_children) |
| signal.signal(signal.SIGTERM, sys_exit) |
| |
| if not args.no_update_repos: |
| print_status('Cleaning and updating local trees...') |
| clean_build_outputs() |
| clean_and_update_repo(local_dart_sdk_repo) |
| clean_and_update_repo(local_flutter_repo) |
| clean_and_update_forked_repo(local_engine_flutter_repo) |
| else: |
| print_warning('Skipping cleaning and updating of local trees') |
| |
| # Use the most recent Dart SDK commit for the roll. |
| if not args.skip_roll: |
| print_status('Starting Dart roll helper') |
| most_recent_commit = '' |
| dart_roll_helper_args = [] |
| if args.skip_update_deps: |
| dart_roll_helper_args.append('--no-update-deps') |
| elif args.dart_sdk_revision != None: |
| most_recent_commit = args.dart_sdk_revision |
| else: |
| # Get the most recent commit that is a reasonable candidate. |
| most_recent_commit = get_most_recent_green_build() |
| if args.skip_tests: |
| dart_roll_helper_args.append('--no-test') |
| if args.skip_build: |
| dart_roll_helper_args.append('--no-build') |
| if args.skip_update_licenses: |
| dart_roll_helper_args.append('--no-update-licenses') |
| if not args.skip_pull_request: |
| dart_roll_helper_args.append('--create-commit') |
| |
| |
| # Will exit with code ERROR_OLD_COMMIT_PROVIDED if `most_recent_commit` is |
| # older than the current revision of the SDK used by Flutter. |
| result = run_dart_roll_helper(most_recent_commit, dart_roll_helper_args) |
| if result != 0: |
| sys.exit(result) |
| else: |
| print_warning('Skipping roll step!') |
| |
| if not args.skip_pull_request: |
| # If the local roll was successful, try to merge into the engine. |
| print_status('Creating flutter/engine pull request') |
| current_date = datetime.datetime.today().strftime('%Y-%m-%d') |
| |
| try: |
| CURRENT_PR = create_pull_request(GITHUB_ENGINE_REPO, |
| local_engine_flutter_repo, |
| get_pr_title(local_engine_flutter_repo), |
| 'dart-sdk-roll-{}'.format(current_date)) |
| except DartAutorollerException as e: |
| print_error(('Error while creating flutter/engine pull request: {}.' |
| ' Aborting roll.').format(e)) |
| sys.exit(1) |
| |
| print_status('Waiting for PR checks to complete...') |
| merge_on_success(GITHUB_ENGINE_REPO, local_engine_flutter_repo, CURRENT_PR) |
| print_status('PR checks complete!') |
| CURRENT_PR = None |
| else: |
| print_warning('Not creating flutter/engine PR!') |
| |
| # TODO(bkonyi): uncomment if we decide to roll the engine into the framework. |
| # print_status('Starting roll of flutter/engine into flutter/flutter') |
| # If the roll into the engine succeeded, prep the roll into the framework. |
| # run_engine_roll_helper(local_engine_flutter_repo, |
| # flutter_path, |
| # local_flutter_repo, |
| # github_flutter_repo) |
| |
| # Status code should be 0 anyway, but let's make sure our exit status is |
| # consistent throughout the tool on a successful roll. |
| sys.exit(ERROR_ROLL_SUCCESS) |
| |
| if __name__ == '__main__': |
| main() |
