#!/usr/bin/python -u
# Copyright 2019 The Dart project authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
#
# This script automates the steps required to fully roll a recent version of the
# Dart SDK into the Flutter engine (and can easily be extended to roll into the
# Flutter framework as well).
#
# The following steps are completed as part of the roll:
#   - The Dart buildbots are queried to determine which SDK revision should be
#     used. Only revisions which have finished all VM and Flutter builds will be
#     considered, and revisions with < 90% of the VM bots green are ignored. See
#     dart_buildbot_helper.py for more details.
#   - dart_roll_helper.py is run with the chosen revision. This performs all the
#     interesting parts of the roll including running tests. If the steps all
#     complete successfully, dart_roll_helper.py exits with code 0. Other
#     possible statuses are listed in dart_roll_utils.py, and will cause this
#     script to exit with a non-zero exit code. dart_roll_helper.py can also be
#     run manually to perform a Dart SDK roll with specific parameters.
#   - The commit created by dart_roll_helper.py is pushed to a branch in
#     flutter/engine and a pull request is created. Once all PR checks are
#     complete, the PR is either merged or closed and all state created by this
#     script is cleaned up.
#
# In order for this script to work, the following environment variables much be
# set:
#   - GITHUB_API_KEY: A GitHub personal access token for the GitHub account to
#     be used for uploading the SDK roll changes (see
#     https://github.com/settings/tokens)
#   - FLUTTER_HOME: the absolute path to the 'flutter' directory
#   - ENGINE_HOME: the absolute path to the 'engine/src' directory
#   - DART_SDK_HOME: the absolute path to the root of a Dart SDK project
#   - ENGINE_FORK: the name of the GitHub fork to use (e.g., bkonyi/engine or
#     flutter/engine)
#
# Finally, the following pip commands need to be run:
#   - `pip install gitpython` for GitPython (git)
#   - `pip install PyGithub` for PyGithub (github)

from dart_buildbot_helper import get_most_recent_green_build
from dart_roll_utils import *
from git import Repo
from github import Github, GithubException
import argparse
import atexit
import datetime
import os
import signal
import shutil
import subprocess
import sys
import time

GITHUB_STATUS_FAILURE = 'failure'
GITHUB_STATUS_PENDING = 'pending'
GITHUB_STATUS_SUCCESS = 'success'

PULL_REQUEST_DESCRIPTION = (
  'This is an automated pull request which will automatically merge once '
  'checks pass.'
)

FLAG_skip_wait_for_artifacts = False

CURRENT_SUBPROCESS = None
CURRENT_PR = None
GITHUB_ENGINE_REPO = None
GITHUB_ENGINE_FORK = None

def run_dart_roll_helper(most_recent_commit, extra_args):
  global CURRENT_SUBPROCESS
  os.environ["PYTHONUNBUFFERED"] = "1"
  args = ['python',
          os.path.join(os.path.dirname(__file__), 'dart_roll_helper.py'),
          '--no-hot-reload',
          most_recent_commit] + extra_args
  CURRENT_SUBPROCESS = subprocess.Popen(args)
  result = CURRENT_SUBPROCESS.wait()
  CURRENT_SUBPROCESS = None
  return result


# TODO(bkonyi): uncomment if we decide to roll into the framework.
# def get_engine_version_path(flutter_repo_path):
#  return os.path.join(flutter_repo_path,
#                      'bin',
#                      'internal',
#                      'engine.version')
#
#
# def get_current_engine_version(flutter_repo_path):
#   with open(get_engine_version_path(flutter_repo_path), 'r') as f:
#     return f.readline().strip()
#
#
# def update_engine_version(flutter_repo_path, sha):
#   with open(get_engine_version_path(flutter_repo_path), 'w') as f:
#     f.write(sha)
#
#
# def run_engine_roll_helper(engine_local_repo,
#                           flutter_repo_path,
#                           flutter_local_repo,
#                           flutter_github_repo):
#  clean_and_update_repo(engine_local_repo)
#  engine_commits  = list(engine_local_repo.iter_commits())[:2]
#  pre_roll_commit = engine_commits[1].hexsha
#  roll_commit     = engine_commits[0].hexsha
#  current_engine_version = get_current_engine_version(flutter_repo_path)
#
#  # Run `flutter doctor` until artifacts are uploaded to cloud.
#  wait_for_engine_artifacts(flutter_repo_path, roll_commit)
#
#  # Update the engine repo again to get any changes that may have gone in while
#  # waiting for the artifacts to build.
#  clean_and_update_repo(engine_local_repo)
#
#  if not is_ancestor_commit(current_engine_version,
#                            roll_commit,
#                            engine_flutter_path()):
#    print_status(('Existing revision {} already contains the Dart SDK roll. '
#                  'No more work to do!').format(current_engine_version))
#    sys.exit(ERROR_ROLL_SUCCESS)
#
#  current_date = datetime.datetime.today().strftime('%Y-%m-%d')
#  branch_name = 'dart-sdk-roll-{}'.format(current_date)
#  engine_version_path = get_engine_version_path(flutter_repo_path)
#  pr_name = 'Dart SDK roll for {}'.format(current_date)
#
#  if pre_roll_commit != current_engine_version:
#    # Update the engine version to the commit before the Dart SDK roll.
#    # This ensures that the Dart SDK version bump is the only change in
#    # the engine roll.
#    update_engine_version(flutter_repo_path, pre_roll_commit)
#    create_commit(flutter_local_repo,
#                  branch_name,
#                  'Roll engine ahead of Dart SDK roll',
#                  [engine_version_path])
#
#  # Actually update the engine version to include the Dart SDK version bump.
#  update_engine_version(flutter_repo_path, roll_commit)
#  create_commit(flutter_local_repo,
#                branch_name,
#                'Roll engine with Dart SDK roll',
#                [engine_version_path])
#
#  pull_request = create_pull_request(flutter_github_repo,
#                                     flutter_local_repo,
#                                     pr_name,
#                                     branch_name)
#
#  merge_on_success(flutter_github_repo, pull_request)
#
#
# def wait_for_engine_artifacts(flutter_repo_path, engine_revision):
#  if FLAG_skip_wait_for_artifacts:
#    print_warning('Skipping wait for Flutter engine artifacts.')
#    return
#
#  flutter_tools = os.path.join(flutter_repo_path, 'bin', 'flutter')
#  cache_path    = os.path.join(flutter_repo_path, 'bin', 'cache')
#
#  # Run `flutter doctor` until it can successfully find the engine artifacts
#  args = [flutter_tools,
#          'doctor',
#          '--check-for-remote-artifacts',
#          engine_revision]
#  while True:
#    result = subprocess.Popen(args, stdout=subprocess.DEVNULL).wait()
#    if result == 0:
#      break
#    time.sleep(15)
#
#
# def create_commit(local_repo, branch, message, files):
#  local_repo.create_head(branch)
#  local_repo.git.checkout(branch)
#  index = local_repo.index
#  index.add(files)
#  index.commit(message)


def clean_and_update_repo(local_repo):
  local_repo.git.checkout('.')
  local_repo.git.clean('-xdf')
  local_repo.git.checkout('master')
  local_repo.git.pull()

def clean_and_update_forked_repo(local_repo):
  local_repo.git.checkout('.')
  local_repo.git.clean('-xdf')
  local_repo.git.fetch('upstream')
  local_repo.git.checkout('master')
  local_repo.git.merge('upstream/master')

def clean_build_outputs():
  print_status('Cleaning build directory...')
  args = ['rm', '-rf',
          os.path.join(ENGINE_HOME, 'out')]
  CURRENT_SUBPROCESS = subprocess.Popen(args)
  CURRENT_SUBPROCESS.wait()
  CURRENT_SUBPROCESS = None

def delete_local_branch(local_repo, branch):
  print_status('Deleting local branch {} in: {}'.format(
      branch,
      local_repo.working_tree_dir))
  local_repo.git.checkout('master')
  local_repo.delete_head(branch, '-D')


def delete_remote_branch(github_repo, branch):
  print_status('Deleting remote branch on {}: {}'.format(github_repo.full_name,
                                                         branch))
  github_repo.get_git_ref('heads/{}'.format(branch)).delete()


def get_most_recent_commit(local_repo):
  commits = list(local_repo.iter_commits())[:1]
  return commits[0]


def get_pr_title(local_repo):
  commit = get_most_recent_commit(local_repo)
  return commit.message.splitlines()[0].rstrip()


def create_pull_request(github_repo, local_repo, title, branch):
  local_repo.create_head(branch)
  local_repo.git.checkout(branch)
  local_repo.git.push('origin', branch)
  commit = get_most_recent_commit(local_repo)
  description = PULL_REQUEST_DESCRIPTION + '\n\n' + commit.message
  try:
    return github_repo.create_pull(title, description,
                                   'master', '{}:{}'.format('bkonyi', branch))
  except GithubException as e:
    delete_remote_branch(GITHUB_ENGINE_FORK, branch)
    raise DartAutorollerException(e.data['errors'][0]['message'])
  finally:
    print_status('Cleaning up local branch: {}'.format(branch))
    delete_local_branch(local_repo, branch)
    # Remove the commit from the local master branch.
    local_repo.git.reset('--hard','origin/master')


def cleanup_pr(github_repo, pull_request, reason):
  msg = '{}, abandoning roll.'.format(reason)
  pull_request.create_issue_comment(msg)
  pull_request.edit(state='closed')
  print_error(msg)
  delete_remote_branch(GITHUB_ENGINE_FORK, pull_request.head.ref)


def merge_on_success(github_repo, local_repo, pull_request):
  sha = pull_request.head.sha
  commit = github_repo.get_commit(sha=sha)

  # TODO(bkonyi): Handle case where Flutter tree is red and we're trying to
  # merge into flutter/flutter.
  should_merge = wait_for_status(pull_request, commit)
  if should_merge:
    if not pull_request.is_merged():
      pull_request.create_issue_comment('Checks successful, automatically merging.')
      merge_status = pull_request.merge(merge_method='squash').merged
      if not merge_status:
        print_error('Merge failed! Aborting roll.')
        sys.exit(1)
      print_status('Merge was successful!')
    else:
      print_status('Manual merge was performed.')
  else:
    cleanup_pr(github_repo, pull_request, 'Checks failed')
    sys.exit(1)
  delete_remote_branch(GITHUB_ENGINE_FORK, pull_request.head.ref)


# TODO(bkonyi): Check to see if the Flutter build is green for flutter/flutter
# if we decide to roll the engine into the framework.
# def flutter_build_passing(commit):
#   FLUTTER_BUILD = 'flutter-build'
#   statuses = commit.get_statuses()
#   for status in statuses:
#     if status.context == FLUTTER_BUILD:
#       return (status.state == GITHUB_STATUS_SUCCESS)
# If flutter-build isn't a valid status, the PR checks don't require the
# Flutter framework to be green to submit.
#   return True

# Determines if any failures are actual failures from the PR or are just
# failures from the engine builders.
def is_only_engine_build_failing(commit):
  BUILD = '-build'
  LUCI_ENGINE = 'luci-engine'
  statuses = commit.get_statuses()
  for status in statuses:
    if (not ((BUILD in status.context) or
             (LUCI_ENGINE == status.context)) and
            (status.state == GITHUB_STATUS_FAILURE)):
        return False
  print_status("An engine builder is still failing...")
  return True


def wait_for_status(pull_request, commit):
  if FLAG_skip_wait_for_artifacts:
    return True

  print_status('Sleeping for 120 seconds to allow for Cirrus to start...')

  # Give Cirrus a chance to start. The GitHub statuses posted by Cirrus go
  # through some weird states when the PR is created and can be marked as
  # failing temporarily, causing this check to return False if we don't wait.
  time.sleep(120)

  print_status('Starting PR status checks (this may take awhile).')

  # Ensure all checks pass.
  while True:
    # Check to see if the PR was manually merged.
    if pull_request.is_merged():
      break
    status = commit.get_combined_status().state
    if status == GITHUB_STATUS_SUCCESS:
      break
    # If the only the engine builders are red, keep trying.
    elif ((status == GITHUB_STATUS_FAILURE) and
          (not is_only_engine_build_failing(commit))):
      return False
    time.sleep(5)

  # TODO(bkonyi): Re-enable this check if we decide to roll the engine into the
  # framework.
  # Once all checks are passing, wait for the Flutter build to be green.
  # while not flutter_build_passing(commit):
  #   print_status('Waiting for Flutter build to pass...')
  #   time.sleep(60)
  # print_status('Flutter build passing!')
  return True


def sys_exit(signal, frame):
  sys.exit()


def cleanup_children():
  print_error('Roll canceled! Shutting down dart_autoroller.py.')
  if CURRENT_SUBPROCESS != None:
    CURRENT_SUBPROCESS.terminate()
  if CURRENT_PR != None:
    cleanup_pr(GITHUB_ENGINE_REPO, CURRENT_PR, 'Canceled by roller')


def main():
  global CURRENT_PR
  global FLAG_skip_wait_for_artifacts
  global GITHUB_ENGINE_REPO
  global GITHUB_ENGINE_FORK

  parser = argparse.ArgumentParser(description='Dart SDK autoroller for Flutter.')
  parser.add_argument('--dart-sdk-revision',
                      help='Provide a Dart SDK revision to roll instead of '
                        'choosing one automatically')
  parser.add_argument('--no-update-repos',
                      help='Skip cleaning and updating local repositories',
                      action='store_true')
  parser.add_argument('--skip-roll',
                      help='Skip running dart_roll_helper.py',
                      action='store_true')
  parser.add_argument('--skip-tests',
                      help='Skip running Flutter tests',
                      action='store_true')
  parser.add_argument('--skip-build',
                      help='Skip building all configurations',
                      action='store_true')
  parser.add_argument('--skip-update-deps',
                      help='Skip updating the Dart SDK dependencies',
                      action='store_true')
  parser.add_argument('--skip-wait-for-artifacts',
                      help="Don't wait for PR statuses to pass or for engine" +
                      " artifacts to be uploaded to the cloud",
                      action='store_true', default=False)
  parser.add_argument('--skip-update-licenses',
                      help='Skip updating the licenses for the Flutter engine',
                      action='store_true')
  parser.add_argument('--skip-pull-request',
                      help="Skip creating a pull request and don't commit",
                      action='store_true')

  args = parser.parse_args()
  FLAG_skip_wait_for_artifacts = args.skip_wait_for_artifacts
  github_api_key = os.getenv('GITHUB_API_KEY')
  dart_sdk_path  = os.getenv('DART_SDK_HOME')
  flutter_path   = os.getenv('FLUTTER_HOME')
  engine_path    = os.getenv('ENGINE_HOME')
  engine_fork    = os.getenv('ENGINE_FORK')
  local_dart_sdk_repo       = Repo(dart_sdk_path)
  local_flutter_repo        = Repo(flutter_path)
  local_engine_flutter_repo = Repo(os.path.join(engine_path, 'flutter'))
  assert(not local_dart_sdk_repo.bare)
  assert(not local_flutter_repo.bare)
  assert(not local_engine_flutter_repo.bare)

  github = Github(github_api_key)
  GITHUB_ENGINE_REPO  = github.get_repo('flutter/engine')
  GITHUB_ENGINE_FORK  = github.get_repo(engine_fork)
  github_flutter_repo = github.get_repo('flutter/flutter')

  atexit.register(cleanup_children)
  signal.signal(signal.SIGTERM, sys_exit)

  if not args.no_update_repos:
    print_status('Cleaning and updating local trees...')
    clean_build_outputs()
    clean_and_update_repo(local_dart_sdk_repo)
    clean_and_update_repo(local_flutter_repo)
    clean_and_update_forked_repo(local_engine_flutter_repo)
  else:
    print_warning('Skipping cleaning and updating of local trees')

  # Use the most recent Dart SDK commit for the roll.
  if not args.skip_roll:
    print_status('Starting Dart roll helper')
    most_recent_commit = ''
    dart_roll_helper_args = []
    if args.skip_update_deps:
      dart_roll_helper_args.append('--no-update-deps')
    elif args.dart_sdk_revision != None:
      most_recent_commit = args.dart_sdk_revision
    else:
      # Get the most recent commit that is a reasonable candidate.
      most_recent_commit = get_most_recent_green_build()
    if args.skip_tests:
      dart_roll_helper_args.append('--no-test')
    if args.skip_build:
      dart_roll_helper_args.append('--no-build')
    if args.skip_update_licenses:
      dart_roll_helper_args.append('--no-update-licenses')
    if not args.skip_pull_request:
      dart_roll_helper_args.append('--create-commit')


    # Will exit with code ERROR_OLD_COMMIT_PROVIDED if `most_recent_commit` is
    # older than the current revision of the SDK used by Flutter.
    result = run_dart_roll_helper(most_recent_commit, dart_roll_helper_args)
    if result != 0:
      sys.exit(result)
  else:
    print_warning('Skipping roll step!')

  if not args.skip_pull_request:
    # If the local roll was successful, try to merge into the engine.
    print_status('Creating flutter/engine pull request')
    current_date = datetime.datetime.today().strftime('%Y-%m-%d')

    try:
      CURRENT_PR = create_pull_request(GITHUB_ENGINE_REPO,
                                       local_engine_flutter_repo,
                                       get_pr_title(local_engine_flutter_repo),
                                       'dart-sdk-roll-{}'.format(current_date))
    except DartAutorollerException as e:
      print_error(('Error while creating flutter/engine pull request: {}.'
                   ' Aborting roll.').format(e))
      sys.exit(1)

    print_status('Waiting for PR checks to complete...')
    merge_on_success(GITHUB_ENGINE_REPO, local_engine_flutter_repo, CURRENT_PR)
    print_status('PR checks complete!')
    CURRENT_PR = None
  else:
    print_warning('Not creating flutter/engine PR!')

  # TODO(bkonyi): uncomment if we decide to roll the engine into the framework.
  # print_status('Starting roll of flutter/engine into flutter/flutter')
  # If the roll into the engine succeeded, prep the roll into the framework.
  # run_engine_roll_helper(local_engine_flutter_repo,
  #                        flutter_path,
  #                        local_flutter_repo,
  #                        github_flutter_repo)

  # Status code should be 0 anyway, but let's make sure our exit status is
  # consistent throughout the tool on a successful roll.
  sys.exit(ERROR_ROLL_SUCCESS)

if __name__ == '__main__':
  main()