# Copyright 2016 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json

from recipe_engine import recipe_api
from recipe_engine.post_process import (
    DoesNotRunRE,
    Filter,
    MustRun,
    MustRunRE,
    StatusSuccess,
    StatusException,
)

DEPS = [
    'bisect_build',
    'build',
    'depot_tools/bot_update',
    'depot_tools/depot_tools',
    'depot_tools/gclient',
    'depot_tools/gitiles',
    'recipe_engine/buildbucket',
    'recipe_engine/cipd',
    'recipe_engine/context',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/platform',
    'recipe_engine/properties',
    'recipe_engine/runtime',
    'recipe_engine/step',
]

DART_GERRIT = 'https://dart.googlesource.com/'

COMMITS_JSON = 'commits.json'
ENGINE_REPO = 'external/github.com/flutter/engine'
FLUTTER_REPO = 'external/github.com/flutter/flutter'
LINEARIZED_REPO = 'linear_sdk_flutter_engine'
SDK_REPO = 'sdk'

ENGINE_REPO_URL = DART_GERRIT + ENGINE_REPO
FLUTTER_REPO_URL = DART_GERRIT + FLUTTER_REPO
LINEARIZED_REPO_URL = DART_GERRIT + LINEARIZED_REPO

PYTHON_VERSION_COMPATIBILITY = "PY3"


def KillTasks(api, engine_src, ok_ret='any'):
  """Kills leftover tasks from previous runs or steps."""
  dart_sdk_dir = engine_src.join('third_party', 'dart')

  # TODO(athom): remove python detection when python3 CL has landed
  python = api.properties.get('python_version', 'python')
  assert (python in ['python', 'python3'])

  api.step(
      'kill processes',
      [python, '-u', dart_sdk_dir.join('tools', 'task_kill.py')],
      ok_ret=ok_ret)


def Build(api, engine_src, config, *targets):
  build_dir = engine_src.join('out/%s' % config)
  ninja_cmd = [api.depot_tools.ninja_path, '-j', '200', '-C', build_dir]
  ninja_cmd.extend(targets)
  api.step('build %s' % ' '.join([config] + list(targets)), ninja_cmd)


def RunGN(api, engine_src, *args):
  gn_cmd = [engine_src.join('flutter/tools/gn')]
  gn_cmd.extend(args)
  # Flutter's gn tool needs ninja in the PATH
  with api.depot_tools.on_path():
    api.step('gn %s' % ' '.join(args), gn_cmd)


def AnalyzeDartUI(api, engine_src):
  with api.context(cwd=engine_src), api.depot_tools.on_path():
    api.step('analyze dart_ui', ['/bin/bash', 'flutter/ci/analyze.sh'])


def TestEngine(api, engine_src):
  with api.context(cwd=engine_src), api.depot_tools.on_path():
    api.step('test engine', ['/bin/bash', 'flutter/testing/run_tests.sh'])


def BuildLinuxAndroidx86(api, engine_src):
  for x86_variant in ['x64', 'x86']:
    RunGN(api, engine_src, '--android', '--android-cpu=' + x86_variant,
          '--no-lto', '--no-prebuilt-dart-sdk')
    Build(api, engine_src, 'android_debug_' + x86_variant)


def BuildLinuxAndroidArm(api, engine_src):
  RunGN(api, engine_src, '--android', '--no-lto', '--no-prebuilt-dart-sdk')
  Build(api, engine_src, 'android_debug')
  Build(api, engine_src, 'android_debug', ':dist')

  # Build engines for the runtime modes that use AOT compilation.
  for arch in ['arm', 'arm64', 'x64']:
    for mode in ['profile', 'release']:
      build_dir = 'android_%s%s' % (mode, '' if arch == 'arm' else '_' + arch)

      RunGN(api, engine_src, '--android', '--runtime-mode=' + mode,
            '--android-cpu=' + arch, '--no-prebuilt-dart-sdk')
      # Build the default set of targets.
      Build(api, engine_src, build_dir)


def BuildLinux(api, engine_src):
  RunGN(api, engine_src, '--full-dart-sdk', '--no-prebuilt-dart-sdk')
  Build(api, engine_src, 'host_debug')
  RunGN(api, engine_src, '--unoptimized', '--no-prebuilt-dart-sdk')
  Build(api, engine_src, 'host_debug_unopt')
  # analyze step needs dart ui sources
  RunGN(api, engine_src, '--runtime-mode=release', '--no-prebuilt-dart-sdk')
  Build(api, engine_src, 'host_release')
  RunGN(api, engine_src, '--fuchsia', '--fuchsia-cpu', 'x64',
        '--runtime-mode=debug', '--no-lto', '--no-prebuilt-dart-sdk')
  Build(api, engine_src, 'fuchsia_debug_x64',
        'flutter/shell/platform/fuchsia:fuchsia', 'fuchsia_tests')


def TestObservatory(api, engine_src):
  flutter_tester_path = engine_src.join('out/host_debug/flutter_tester')
  empty_main_path = engine_src.join(
      'flutter/shell/testing/observatory/empty_main.dart')
  test_path = engine_src.join('flutter/shell/testing/observatory/test.dart')
  test_cmd = ['dart', test_path, flutter_tester_path, empty_main_path]
  with api.context(cwd=engine_src):
    # Timeout after 5 minutes, this step is prone to hang
    api.step('test observatory and service protocol', test_cmd, timeout=5*60)


def GetJDK(api, start_dir):
  """Downloads OpenJdk CIPD package and updates environment variables.

    Args:
      start_dir(Path): Where to install the JDK directory
    """
  version = 'version:1.8.0u202-b08'
  with api.step.nest('OpenJDK dependency'):
    java_home = start_dir.join('java')
    api.file.ensure_directory('mkdir engine', java_home)
    api.cipd.ensure(
        java_home,
        api.cipd.EnsureFile().add_package(
            'flutter_internal/java/openjdk/${platform}', version))
  return java_home


def GetCheckout(api, start_dir, engine_src):
  src_cfg = api.gclient.make_config()
  src_cfg.target_os = set(['android'])
  commits = {}

  # tryjobs don't have a gitiles_commit.
  if api.buildbucket.gitiles_commit.id:
    commits = json.loads(
        api.gitiles.download_file(
            LINEARIZED_REPO_URL,
            COMMITS_JSON,
            api.buildbucket.gitiles_commit.id,
            step_test_data=lambda: api.gitiles.test_api.make_encoded_file(
                json.dumps({
                    ENGINE_REPO: 'eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee',
                    SDK_REPO: 'dddddddddddddddddddddddddddddddddddddddd'
                }))))
  engine_rev = commits.get(ENGINE_REPO, 'HEAD')
  flutter_rev = commits.get(FLUTTER_REPO, 'HEAD')
  sdk_rev = commits.get(SDK_REPO, 'HEAD')

  src_cfg.revisions = {
      'src/flutter': engine_rev,
      'src/third_party/dart': sdk_rev,
      '../flutter': flutter_rev,
  }

  soln = src_cfg.solutions.add()
  soln.name = 'src/flutter'
  soln.url = ENGINE_REPO_URL
  soln.custom_vars['download_dart_sdk'] = False

  soln = src_cfg.solutions.add()
  soln.name = '../flutter'
  soln.url = FLUTTER_REPO_URL

  api.gclient.c = src_cfg
  api.bot_update.ensure_checkout(
      add_blamelists=True,
      ignore_input_commit=True,
      set_output_commit=False,
      update_presentation=True)
  properties = api.step.active_result.presentation.properties
  properties['rev_engine'] = engine_rev
  properties['rev_flutter'] = flutter_rev
  properties['rev_sdk'] = sdk_rev
  properties['got_revision'] = api.buildbucket.gitiles_commit.id

  with api.depot_tools.on_path():
    # TODO(athom): remove symlink once 3xH tools are updated in the SDK repo.
    api.file.symlink('symlink flutter to new checkout location',
                     start_dir.join('flutter'),
                     start_dir.join('engine', 'flutter'))
    api.step(
        '3xHEAD Flutter Hooks',
        [engine_src.join('third_party/dart/tools/3xhead_flutter_hooks.sh')])

  api.gclient.runhooks()

  return flutter_rev


def CopyArtifacts(api, engine_src, cached_dest, file_paths):
  # cached_dest folder might not exist: flutter update-packages downloads only
  # artifacts that are needed by the connected devices and 3xHEAD bot
  # does not have any devices attached.
  api.file.ensure_directory('mkdir %s' % cached_dest, cached_dest)
  for path in file_paths:
    source, target = path, api.path.basename(path)

    api.file.remove('remove %s' % target, cached_dest.join(target))
    api.file.copy('copy %s' % target, engine_src.join(source),
                  cached_dest.join(target))


def UpdateCachedEngineArtifacts(api, flutter, engine_src):
  ICU_DATA_PATH = 'third_party/icu/flutter/icudtl.dat'
  CopyArtifacts(
      api, engine_src,
      flutter.join('bin', 'cache', 'artifacts', 'engine', 'linux-x64'), [
          ICU_DATA_PATH,
          'out/host_debug_unopt/flutter_tester',
          'out/host_debug_unopt/gen/flutter/lib/snapshot/isolate_snapshot.bin',
          'out/host_debug_unopt/gen/flutter/lib/snapshot/vm_isolate_snapshot.bin',
          'out/host_debug_unopt/gen/frontend_server.dart.snapshot',
          'out/host_debug_unopt/gen/const_finder.dart.snapshot',
      ])

  # Copy over new versions of gen_snapshot for profile/release arm/arm64
  for arch in ['arm', 'arm64', 'x64']:
    for mode in ['profile', 'release']:
      build_dir = 'android_%s%s' % (mode, '' if arch == 'arm' else '_' + arch)
      CopyArtifacts(
          api, engine_src,
          flutter.join('bin', 'cache', 'artifacts', 'engine',
                       'android-%s-%s' % (arch, mode), 'linux-x64'),
          ['out/%s/clang_x64/gen_snapshot' % build_dir])

  flutter_patched_sdk = flutter.join('bin', 'cache', 'artifacts', 'engine',
                                     'common', 'flutter_patched_sdk')
  flutter_patched_sdk_product = flutter.join('bin', 'cache', 'artifacts',
                                             'engine', 'common',
                                             'flutter_patched_sdk_product')
  dart_sdk = flutter.join('bin', 'cache', 'dart-sdk')
  flutter_web_sdk = flutter.join('bin', 'cache', 'flutter_web_sdk')
  pkg = flutter.join('bin', 'cache', 'pkg')
  # In case dart-sdk symlink was left from previous run we need to [remove] it,
  # rather than [rmtree] because rmtree is going to remove symlink target
  # folder. We are not able to use api.file classes for this because there is
  # no support for symlink checks or handling of error condition.
  api.step('cleanup dart-sdk', [
    '/bin/bash', '-c',
    'if [ -L "%(dir)s" ]; then rm "%(dir)s"; else rm -rf "%(dir)s"; fi' %
    {'dir': dart_sdk}])
  api.step('cleanup flutter_web_sdk', [
      '/bin/bash', '-c',
      'if [ -L "%(dir)s" ]; then rm "%(dir)s"; else rm -rf "%(dir)s"; fi' % {
          'dir': flutter_web_sdk
      }
  ])
  api.step('cleanup pkg', [
      '/bin/bash', '-c',
      'if [ -L "%(dir)s" ]; then rm "%(dir)s"; else rm -rf "%(dir)s"; fi' % {
          'dir': pkg
      }
  ])
  api.step('cleanup flutter_patched_sdk', [
    '/bin/bash', '-c',
    'if [ -L "%(dir)s" ]; then rm "%(dir)s"; else rm -rf "%(dir)s"; fi' %
    {'dir': flutter_patched_sdk}])
  api.step('cleanup flutter_patched_sdk_product', [
    '/bin/bash', '-c',
    'if [ -L "%(dir)s" ]; then rm "%(dir)s"; else rm -rf "%(dir)s"; fi' %
    {'dir': flutter_patched_sdk_product}])
  api.file.symlink('symlink just built dart sdk to cached location',
                   engine_src.join('out', 'host_debug', 'dart-sdk'), dart_sdk)
  api.file.symlink('symlink just built flutter web sdk to cached location',
                   engine_src.join('out', 'host_debug', 'flutter_web_sdk'),
                   flutter_web_sdk)
  api.file.symlink('symlink just built pkg from dart sdk to cached location',
                   engine_src.join('out', 'host_debug', 'gen', 'dart-pkg'), pkg)
  api.file.symlink(
    'make cached flutter_patched_sdk point to just built flutter_patched_sdk',
    engine_src.join('out', 'host_debug', 'flutter_patched_sdk'),
    flutter_patched_sdk)
  api.file.symlink(
    'make cached flutter_patched_sdk_product point to just built release '
    'version of flutter_patched_sdk',
    engine_src.join('out', 'host_release', 'flutter_patched_sdk'),
    flutter_patched_sdk_product)

  # In case there is a cached version of "flutter_tools.snapshot" we have to
  # delete it.
  flutter_tools_snapshot = flutter.join(
      'bin', 'cache', 'flutter_tools.snapshot')
  api.step('cleanup', [
    '/bin/bash', '-c', 'if [ -f "%(file)s" ]; then rm "%(file)s"; fi' %
    {'file': flutter_tools_snapshot}])


def TestFlutter(api, start_dir, engine_src, just_built_dart_sdk):
  flutter = start_dir.join('flutter')
  flutter_cmd = flutter.join('bin/flutter')
  test_args = [
      '--local-engine=host_debug',
      '--local-engine-src-path=%s' % engine_src,
  ]
  test_cmd = [
    'dart', 'dev/bots/test.dart',
  ]
  api.step('disable flutter analytics', [
      flutter_cmd, 'config', '--no-analytics'])
  with api.context(cwd=flutter):

    # Precache so that later flutter won't overwrite
    # updated artifacts.
    api.step('flutter precache', [flutter_cmd, 'precache'])

    # analyze.dart and test.dart have hardcoded references to
    # bin/cache/dart-sdk.  So we overwrite bin/cache/dart-sdk and
    # tightly-coupled frontend_server.dart.snapshot with links that point to
    # corresponding entries from binaries generated into [engine_src]
    UpdateCachedEngineArtifacts(api, flutter, engine_src)

    api.step('flutter update-packages',
             [flutter_cmd, 'update-packages'] + test_args)
    all_test_suites = [
        'add_to_app_life_cycle_tests',
        'build_tests',
        'flutter_plugins',
        'framework_coverage',
        'framework_tests',
        'tool_tests',
        'tool_integration_tests',
        'web_tool_tests',
        'web_tests',
    ]
    # The property that selects which test suites to run is configured in
    # https://dart.googlesource.com/sdk/+/refs/heads/infra/config/main.star.
    test_suites = api.properties.get('flutter_test_suites', all_test_suites)
    for test_suite in test_suites:
      if not test_suite in all_test_suites:
        raise api.step.InfraFailure('invalid test suite name "%s"' % test_suite)
    api.step(
        'flutter analyze', [
            'dart', '--enable-asserts', 'dev/bots/analyze.dart', '--dart-sdk',
            just_built_dart_sdk
        ],
        timeout=20 * 60)  # 20 minutes

    with api.context(
        env={
            'ENGINE_PATH': start_dir.join('engine'),
            'WEB_UI_DIR': engine_src.join('flutter', 'lib', 'web_ui')
        }):
      api.step(
          'flutter web engine analyze', [
              engine_src.join('flutter', 'lib', 'web_ui', 'dev',
                              'web_engine_analysis.sh')
          ],
          timeout=20 * 60)  # 20 minutes

    # Runs all flutter tests as described on this page:
    # https://github.com/flutter/flutter/blob/master/CONTRIBUTING.md
    for test_suite in test_suites:
      with api.context(
          env={
              'SHARD':
                  test_suite,
              # flutter-plugins expects to find flutter and pub executables in the path
              'PATH':
                  api.path.pathsep.join((str(flutter.join('bin')),
                                         str(just_built_dart_sdk.join('bin')),
                                         '%(PATH)s')),
          }):
        api.step(
            'flutter test %s' % (test_suite),
            test_cmd + test_args,
            timeout=120 * 60)  # 2 hours


def RunSteps(api):
  start_dir = api.path['cache'].join('builder')
  engine_dir = start_dir.join('engine')
  engine_src = engine_dir.join('src')

  try:
    if api.properties.get('clobber', False):
      api.file.rmcontents('everything', start_dir)
    java_home = GetJDK(api, start_dir)
    with api.context(
        env={'JAVA_HOME': str(java_home)},
        env_prefixes={'PATH': [java_home.join('bin')]}):
      api.file.ensure_directory('mkdir engine', engine_dir)
      with api.context(cwd=engine_dir):
        flutter_rev = GetCheckout(api, start_dir, engine_src)

      KillTasks(api, engine_src)

      BuildAndTest(api, start_dir, engine_src, flutter_rev)
  except recipe_api.StepFailure as failure:
    if api.bisect_build.is_enabled and api.buildbucket.gitiles_commit.id:
      api.bisect_build.schedule(LINEARIZED_REPO_URL, failure.reason)
    raise
  finally:
    # TODO(aam): Go back to `ok_ret={0}` once dartbug.com/35549 is fixed
    KillTasks(api, engine_src, ok_ret='any')
  if api.bisect_build.is_bisecting():
    # The build was successful, so search newer builds to find the root cause.
    api.bisect_build.schedule(LINEARIZED_REPO_URL,
                              api.m.bisect_build.REASON_SUCCESS)


def BuildAndTest(api, start_dir, engine_src, flutter_rev):
  run_env = {
      # By setting 'ANALYZER_STATE_LOCATION_OVERRIDE' we force analyzer to emit
      # its cached state into the given folder. If something goes wrong with
      # the cache we request a clobber using the buildbucket 'bb' command.
      'ANALYZER_STATE_LOCATION_OVERRIDE': start_dir.join('.dartServer'),
      'ANDROID_SDK_ROOT': engine_src.join('third_party', 'android_tools', 'sdk')
  }
  with api.context(cwd=start_dir, env=run_env):
    api.step('accept android licenses', [
        'bash', '-c', 'yes | $ANDROID_SDK_ROOT/tools/bin/sdkmanager --licenses'
    ])

    with api.build.goma():
      BuildLinux(api, engine_src)
    prebuilt_dart_bin = engine_src.join('third_party', 'dart', 'tools', 'sdks',
                                        'dart-sdk', 'bin')
    engine_env = {
        'PATH': api.path.pathsep.join((str(prebuilt_dart_bin), '%(PATH)s')),
    }
    just_built_dart_sdk = engine_src.join('out', 'host_debug', 'dart-sdk')

    # The web test shards need google-chrome installed.
    # The chrome version is the version of a package at:
    # http://go/cipd/p/dart/browsers/chrome/linux-amd64/
    chrome_version = '101.0.4951.41'
    browser_dir = _download_browser(api, 'chrome', chrome_version)
    chrome_executable = browser_dir.join('chrome', 'google-chrome')
    flutter_env = {
        'PATH':
            api.path.pathsep.join(
                (str(just_built_dart_sdk.join('bin')), '%(PATH)s')),
        # Prevent test.dart from using git merge-base to determine a fork point.
        # git merge-base doesn't work without a FETCH_HEAD, which isn't
        # available on the first run of a bot. The builder tests a single
        # revision, so use flutter_rev.
        'TEST_COMMIT_RANGE':
            flutter_rev,
        'CHROME_EXECUTABLE':
            chrome_executable,
        'WEB_SHARD_COUNT':
            '16',
        'FLUTTER_SKIP_ENGINE_CHECK':
            'true',
    }

    with api.step.defer_results():
      # The context adds prebuilt dart-sdk to the path.
      with api.context(env=engine_env):
        AnalyzeDartUI(api, engine_src)
        TestEngine(api, engine_src)
        TestObservatory(api, engine_src)
        with api.build.goma():
          BuildLinuxAndroidArm(api, engine_src)
          BuildLinuxAndroidx86(api, engine_src)
      # The context adds freshly-built engine's dart-sdk to the path.
      with api.context(env=flutter_env):
        TestFlutter(api, start_dir, engine_src, just_built_dart_sdk)


def _download_browser(api, runtime, version):
  # Download CIPD package
  #  dart/browsers/<runtime>/${platform} <version>
  # to directory
  #  [sdk root]/third_party/browsers
  browser_path = api.path['checkout'].join('third_party', 'browsers', runtime)
  api.file.ensure_directory('create browser cache', browser_path)
  version_tag = 'version:%s' % version
  package = 'dart/browsers/%s/${platform}' % runtime
  ensure_file = api.cipd.EnsureFile().add_package(package, version_tag)
  api.cipd.ensure(browser_path, ensure_file)
  return browser_path


def _test(api, name, failure=False):
  data = api.test(
      name,
      api.platform('linux', 64),
      api.buildbucket.ci_build(
          builder='flutter-engine-linux',
          git_ref='refs/heads/master',
          git_repo=LINEARIZED_REPO_URL,
          build_number=3,
          revision='f' * 8),
      api.properties(bot_id='fake-m1', clobber=True, bisection_enabled=True),
      api.runtime(is_experimental=False),
  )
  if failure:
    # let the first step in the recipe fail
    data += api.step_data('everything', retcode=1)
  return data


def GenTests(api):
  yield (_test(api, 'only-run-webtests-suite') +
         api.properties(flutter_test_suites=['web_tests']) +
         api.post_process(MustRun, 'flutter test web_tests') +
         api.post_process(StatusSuccess))

  yield (_test(api, 'run-all-test-suites')
         + api.post_process(MustRun, 'flutter test web_tests') +
         api.post_process(StatusSuccess))

  yield (_test(api, 'run-undefined-test-suite') +
         api.properties(flutter_test_suites=['not-a-test-suite']) +
         api.post_process(StatusException))

  yield (_test(api, 'flutter-engine-linux')
      + api.post_process(DoesNotRunRE, r'schedule bisect.*'))

  yield (_test(api, 'start-bisect', failure=True) +
         api.bisect_build.fetch_previous_builds([
             api.bisect_build.build("foo", 4710,
                                    "2d72510e447ab60a9728aeea2362d8be2cbd7789")
         ]) + api.step_data(
             'gitiles log: 2d72510e447ab60a9728aeea2362d8be2cbd7789..ffffffff',
             api.gitiles.make_log_test_data('master')) +
         api.post_process(MustRunRE, 'schedule bisect .*'))

  yield (_test(api, 'continue-bisect-on-success') + api.properties(
      bisect_newer=['a', 'b', 'c'],
      bisect_older=['c', 'd', 'e'],
      bisect_base_build=4711,
      bisect_reason="Infra Failure: Step('everything') (retcode: 1)") +
         api.post_process(MustRun, 'schedule bisect (b)') +
         api.post_process(Filter('schedule bisect (b)')))