# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


from recipe_engine.post_process import (
    DoesNotRun,
    DoesNotRunRE,
    DropExpectation,
    Filter,
    MustRun,
    MustRunRE,
    StatusException,
    StatusFailure,
    StatusSuccess,
    StepCommandContains,
    StepException,
    StepFailure,
)

DEPS = [
    'dart',
    'build',
    'depot_tools/tryserver',
    'recipe_engine/buildbucket',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/platform',
    'recipe_engine/raw_io',
    'recipe_engine/runtime',
    'recipe_engine/step',
    'recipe_engine/swarming',
]


CANNED_OUTPUT_DIR = {
    'logs.json':
        '{"test":"test1", "configuration": "example_configuration"}\n',
    'results.json':
        '{"name":"test1", "configuration": example_configuration"}\n'
        '{"name":"test2", "configuration": example_configuration_2"}\n',
}

CANNED_FLAKY_OUTPUT_DIR = {
    'logs.json':
        '{"name": "Flaky/Test/1", "configuration": "example_configuration"}\n'
        '{"name": "Flaky/Test/2", "configuration": "example_configuration"}\n',
    'results.json':
        '{"name":"Flaky/Test/1", "configuration": "example_configuration"}\n'
        '{"name":"Flaky/Test/2", "configuration": "example_configuration"}\n',
}


TRIGGER_RESULT = {
  "results": [
    {
      "build": {
        "id": "8963024236039183808",
        "tags": [
          "builder:analyzer-linux-release",
          "parent_buildername:dart-sdk-linux"
        ],
        "url": "https://ci.chromium.org/p/dart/builds/b8963024236039183808",
      }
    },
    {
      "build": {
        "id": "8963024236039836208",
        "tags": [
          "builder:analyzer-strong-linux-release",
          "parent_buildername:dart-sdk-linux"
        ],
        "url": "https://ci.chromium.org/p/dart/builds/b8963024236039836208",
      }
    },
    {
      "build": {
        "id": "8963024236039228096",
        "tags": [
          "builder:analyzer-analysis-server-linux",
          "parent_buildername:dart-sdk-linux"
        ],
        "url": "https://ci.chromium.org/p/dart/builds/b8963024236039228096",
      }
    }
  ]
}


TEST_MATRIX = {
    "filesets": {
        "test": "[]",
        "trigger": "[]"
    },
    "global": {
        "chrome": "66.0.3359.139",
        "firefox": "61"
    },
    "builder_configurations": [{
        "builders": ["vm-kernel-win-release-simarm_x64",],
        "meta": {},
        "steps": [{
            "name":
                "gn",
            "script":
                "tools/gn.py",
            "arguments": [
                "--bytecode", "--gen-snapshot=${build_root}/gen_snapshot"
            ]
        }, {
            "name": "build",
            "script": "tools/build.py",
            "arguments": ["--arch=x64", "runtime"]
        }, {
            "name": "test1",
            "script": "tools/test.py",
            "arguments": [
                "foo", "-ndartk-${system}-${mode}-${arch}", "language_2",
                "co19_2/sel", "--exclude_suite=co19"
            ],
            "shards": 2,
            "fileset": "test"
        }, {
            "name": "test2",
            "arguments": ["foo", "--bar", "-ndartk-${system}-${mode}-${arch}"],
        }, {
            "name": "trigger",
            "fileset": "trigger",
            "trigger": ["foo-builder", "bar-builder"]
        }, {
            "name": "dart",
            "script": "out/ReleaseX64/dart",
            "arguments": ["--bar", "foo.dart"]
        }, {
            "name": "test3",
            "arguments": ["-ndartk-${system}-${mode}-${arch}", "foo", "--bar"],
            "fileset": "test",
            "shards": 2
        }]
    }, {
        "builders": ["co19",],
        "meta": {},
        "steps": [{
            "name": "co19",
            "arguments": ["co19/nnbd", "co19_2"],
        }]
    }, {
        "builders": ["co19_sharded",],
        "meta": {},
        "steps": [{
            "name": "co19",
            "arguments": ["co19/nnbd", "co19_2"],
            "fileset": "test",
            "shards": 1
        }]
    }, {
        "builders": ["analyzer-linux-release",],
        "meta": {},
        "steps": [{
            "name": "build",
            "script": "tools/build.py"
        }, {
            "name": "test1",
            "script": "tools/test.py",
            "fileset": "test",
            "shards": 2,
            "arguments": ["-nunittest-asserts-${mode}-${system}"]
        }, {
            "name": "test2",
            "arguments": ["foo", "--bar", "-ndartk-${system}-${mode}-${arch}"],
        }, {
            "name": "trigger",
            "fileset": "trigger",
            "trigger": ["foo-builder", "bar-builder"]
        }, {
            "name": "test3",
            "arguments": ["-nanalyzer-asserts-${system}", "foo", "--bar"],
            "fileset": "test",
            "shards": 2
        }]
    }, {
        "builders": [
            "dart2js-strong-mac-x64-chrome", "dart2js-strong-linux-x64-firefox",
            "dart2js-strong-win-x64-chrome"
        ],
        "meta": {},
        "steps": [{
            "name": "test1",
            "script": "tools/test.py",
            "arguments": [
                "-ndart2js-${system}-${runtime}", "foo", "--bar",
                "-e co19, language_2"
            ],
            "shards": 2,
            "fileset": "test"
        }, {
            "name": "custom",
            "script": "tools/custom_thing.py",
            "arguments": ["foo", "--bar", "--buildername"]
        }, {
            "name": "custom_runner",
            "script": "tools/custom_test_runner.py",
            "testRunner": True,
            "arguments": ["foo", "--bar", "--buildername"]
        }, {
            "name":
                "test2",
            "arguments": [
                "-ndart2js-${system}-${runtime}", "foo", "--bar", "co19"
            ],
        }]
    }, {
        "builders": ["vm-kernel-mac-release-x64"],
        "meta": {},
        "steps": [{
            "name": "build",
            "script": "tools/build.py",
            "arguments": []
        }, {
            "name": "custom",
            "script": "out/custom_thing",
            "arguments": ["foo", "--bar", "--buildername"]
        }]
    }, {
        "builders": ["vm-kernel-precomp-android-release-arm64"],
        "meta": {},
        "steps": [{
            "name": "android",
            "shards": 2,
            "fileset": "test",
            "arguments": ["-ndartkp-android-${mode}-${arch}"]
        }]
    }, {
        "builders": ["fuzz-linux"],
        "steps": [{
            "name": "make a fuzz",
            "script": "out/ReleaseX64/dart",
            "arguments": [
                "runtime/tools/dartfuzz/dartfuzz_test.dart", "--isolates", "8",
                "--no-show-stats", "--time", "2700"
            ],
            "shards": 3,
            "fileset": "test"
        }]
    }]
}

RESULT_DATA = (
    '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t01"'
    ',"configuration":"dartk-linux-product-x64","suite":"co19_2",'
    '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t01",'
    '"time_ms":451,"result":"CompileTimeError","expected":"CompileTimeError",'
    '"matches":true,"commit_time":1551185312,'
    '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",'
    '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,'
    '"previous_flaky":false,"previous_result":"CompileTimeError",'
    '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"previous_commit_time":1551185312,"previous_build_number":2403,'
    '"changed":false}\n' +
    '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t02"'
    ',"configuration":"dartk-linux-product-x64","suite":"co19_2",'
    '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t02",'
    '"time_ms":496,"result":"CompileTimeError","expected":"CompileTimeError",'
    '"matches":true,"commit_time":1551185312,'
    '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",'
    '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,'
    '"previous_flaky":false,"previous_result":"CompileTimeError",'
    '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"previous_commit_time":1551185312,"previous_build_number":2403,'
    '"changed":false}\n')

RESULT_DATA_MULTIPLE_CONFIGURATIONS = (
    '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t01"'
    ',"configuration":"dartk-linux-product-x64","suite":"co19_2",'
    '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t01",'
    '"time_ms":451,"result":"CompileTimeError","expected":"CompileTimeError",'
    '"matches":true,"commit_time":1551185312,'
    '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",'
    '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,'
    '"previous_flaky":false,"previous_result":"CompileTimeError",'
    '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"previous_commit_time":1551185312,"previous_build_number":2403,'
    '"changed":false}\n' +
    '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t02"'
    ',"configuration":"dartk-linux-release-x64","suite":"co19_2",'
    '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t02",'
    '"time_ms":496,"result":"CompileTimeError","expected":"CompileTimeError",'
    '"matches":true,"commit_time":1551185312,'
    '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",'
    '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,'
    '"previous_flaky":false,"previous_result":"CompileTimeError",'
    '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"previous_commit_time":1551185312,"previous_build_number":2403,'
    '"changed":false}\n')


def RunSteps(api):
  api.dart.checkout(
      api.properties.get('clobber', False),
      api.properties.get('custom_vars', {}))

  api.dart.kill_tasks()
  api.dart.read_debug_log()

  api.dart.test(test_data=TEST_MATRIX)


def _canned_step(api, name, shards=0, local_shard=True, deflake=True):
  step_name = '%s_shard_%s' % (name, 1) if shards > 0 else name
  data = None
  for i in range(1 if shards > 0 else 0, shards + 1):
    if not data:
      data = _canned_output_dir(api, step_name)
    else:
      data += _canned_output_dir(api, '%s_shard_%s' % (name, i))
    if deflake:
      deflaking_name = name if local_shard else '%s_shard_1' % name
      data += _canned_output_dir(
          api, deflaking_name, CANNED_FLAKY_OUTPUT_DIR, prefix='deflaking.')
  if deflake:
    data += api.step_data('deflaking.list tests to deflake (%s)' % name,
                stdout=api.raw_io.output('Flaky/Test/1\nFlaky/Test/2'))

  return data


def _canned_output_dir(api, step_name, content=None, prefix=''):
  content = content or CANNED_OUTPUT_DIR
  data = api.empty_test_data()
  for filename in ('logs.json', 'results.json'):
    data += api.step_data('%sread %s for %s' % (prefix, filename, step_name),
                          api.file.read_text(text_content=content[filename]))
  return data


def _bot_script(name):
  return '[CACHE]/builder/sdk/tools/bots/%s' % name


def _upload_isolate_step_data(api, fileset='test'):
  return api.step_data(
      'upload testing fileset %s' % fileset,
      stdout=api.raw_io.output('uploaded digest: %s_hash/47' % fileset))


def GenTests(api):
  UPDATE_BLAMELIST_STEP_RE = ".*blamelist.*"
  UPDATE_BLAMELIST_STEP = ('update blamelists for configuration '
                           '"dartk-linux-product-x64"')

  yield api.test(
      'basic',
      api.properties(
          shard_timeout='600',
          custom_vars={'download_firefox': True},
          python_version='python3'),
      api.buildbucket.try_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox-try',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'custom_runner', deflake=False),
      _upload_isolate_step_data(api),
      api.tryserver.gerrit_change_target_ref('refs/heads/stable'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text('123', name='latest')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(DoesNotRunRE, UPDATE_BLAMELIST_STEP_RE),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'branch-builder-does-not-publish-results',
      api.properties(shard_timeout='600'),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart',
          git_ref="refs/heads/some-branch"),
      _canned_step(api, 'co19'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text('123', name='latest')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(MustRunRE, '.*.gsutil upload results.json'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       [_bot_script('compare_results.dart')]),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'ci-test-data-branch-builder-does-publish-results',
      api.properties(shard_timeout='600'),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart',
          git_ref="refs/heads/ci-test-data"),
      _canned_step(api, 'co19'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text('123', name='latest')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(MustRunRE, '.*.gsutil upload results.json'),
      api.post_process(MustRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       [_bot_script('get_builder_status.dart')]),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'release-builder-does-not-publish-results',
      api.properties(shard_timeout='600'),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='co19-dev',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart',
          git_ref="refs/heads/dev"),
      _canned_step(api, 'co19'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text('123', name='latest')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(MustRunRE, '.*.gsutil upload results.json'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       [_bot_script('compare_results.dart')]),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'bisection-does-not-refer-to-latest-build-or-publish-results',
      api.properties(
          shard_timeout='600', bisect_reason="FAILURE", bisect_base_build=4711),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'custom_runner', deflake=False),
      _upload_isolate_step_data(api),
      api.step_data(
          'add fields to result records',
          api.raw_io.output_text(RESULT_DATA_MULTIPLE_CONFIGURATIONS)),
      api.post_process(DoesNotRunRE, '.*find latest build'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       [_bot_script('compare_results.dart')]),
      api.post_process(MustRun, UPDATE_BLAMELIST_STEP),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'bisection-on-test-branch-updates-blamelist',
      api.properties(
          shard_timeout='600', bisect_reason="FAILURE", bisect_base_build=4711),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart',
          git_ref="refs/heads/ci-test-data"),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'custom_runner', deflake=False),
      _upload_isolate_step_data(api),
      api.step_data(
          'add fields to result records',
          api.raw_io.output_text(RESULT_DATA_MULTIPLE_CONFIGURATIONS)),
      api.post_process(DoesNotRunRE, '.*find latest build'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       [_bot_script('compare_results.dart')]),
      # Verify that the test branch builder updates the staging database
      api.post_process(StepCommandContains, UPDATE_BLAMELIST_STEP, ['-s']),
      api.post_process(StatusSuccess),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'experimental-bisection-build-does-not-update-blamelist',
      api.properties(
          shard_timeout='600', bisect_reason="FAILURE", bisect_base_build=4711),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.runtime(is_experimental=True),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'custom_runner', deflake=False),
      _upload_isolate_step_data(api),
      api.step_data(
          'add fields to result records',
          api.raw_io.output_text(RESULT_DATA_MULTIPLE_CONFIGURATIONS)),
      api.post_process(DoesNotRunRE, '.*find latest build'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       [_bot_script('compare_results.dart')]),
      api.post_process(DoesNotRunRE, UPDATE_BLAMELIST_STEP_RE),
      api.post_process(StatusSuccess),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'analyzer-linux-release',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'test3', 2),
      _upload_isolate_step_data(api, 'test'),
      _upload_isolate_step_data(api, 'trigger'),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(StatusSuccess),
  )

  def expect_no_side_effects(name,
                             builder='analyzer-linux-release',
                             is_experimental=True):
    return api.test(
        name,
        api.properties(bot_id='trusty-dart-123'),
        api.buildbucket.ci_build(
            revision='3456abce78ef',
            build_number=1357,
            builder=builder,
            git_repo='https://dart.googlesource.com/sdk',
            project='dart'),
        _canned_step(api, 'test1', 2, False),
        _canned_step(api, 'test2'),
        _canned_step(api, 'test3', 2),
        api.runtime(is_experimental=is_experimental),
        _upload_isolate_step_data(api, 'test'),
        _upload_isolate_step_data(api, 'trigger'),
        api.step_data(
            'buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
        api.step_data('add fields to result records',
                      api.raw_io.output_text(RESULT_DATA)),
        api.post_process(DoesNotRunRE, r'.*publish results.*'),
        api.post_process(StatusSuccess),
        api.post_process(DropExpectation),
    )

  yield expect_no_side_effects('experimental-builds-are-side-effect-free')
  yield expect_no_side_effects(
      'experimental-channel-builds-are-side-effect-free',
      builder='analyzer-linux-release-experimental',
      is_experimental=False)
  yield expect_no_side_effects(
      'experimental-experimental-channel-builds-are-side-effect-free',
      builder='analyzer-linux-release-experimental')

  yield api.test(
      'build-failure-in-matrix',
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('build', retcode=1),
      api.post_process(DoesNotRun, 'test1'),
      api.post_process(StepFailure, 'build'),
      api.post_process(StatusFailure),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'basic-missing-name',
      api.buildbucket.ci_build(
          builder='this-name-does-not-exist-in-test-matrix',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.post_process(StatusException),
  )


  yield api.test(
      'failed-tests',
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19_sharded',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'co19', 1, False),
      _upload_isolate_step_data(api),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data('test results', retcode=1),
      api.post_process(StepFailure, 'test results'),
      api.post_process(StatusFailure),
      api.post_process(Filter('test results')),
  )

  yield api.test(
      'failed-to-get-test-results',
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19_sharded',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'co19', 1, False),
      _upload_isolate_step_data(api),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data('test results', retcode=2),
      api.post_process(StepException, 'test results'),
      api.post_process(StatusException),
      api.post_process(Filter('test results')),
  )

  yield api.test(
      'vm-win',
      api.platform('win', 64),
      api.properties(clobber=True, bot_id='win-dart-123'),
      api.path.exists(api.path['checkout'].join('out')),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='vm-kernel-win-release-simarm_x64',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _upload_isolate_step_data(api, 'test'),
      _upload_isolate_step_data(api, 'trigger'),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'basic-mac',
      api.platform('mac', 64),
      api.properties(bot_id='mac-dart-123'),
      api.buildbucket.ci_build(
          revision='a' * 40,
          build_number=1357,
          builder='dart2js-strong-mac-x64-chrome-dev',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.properties(clobber=True, custom_vars={'download_chrome': True}),
      api.path.exists(api.path['checkout'].join('xcodebuild')),
      _upload_isolate_step_data(api),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'example-mac',
      api.platform('mac', 64),
      api.buildbucket.ci_build(
          builder='vm-kernel-mac-release-x64',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'example-android',
      api.platform('linux', 64),
      api.buildbucket.ci_build(
          builder='vm-kernel-precomp-android-release-arm64',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'android', 2, False),
      _upload_isolate_step_data(api),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'non-test-step-error-is-step-failure',
      api.buildbucket.ci_build(
          builder='vm-kernel-win-release-simarm_x64',
          git_repo='https://dart.googlesource.com/sdk'),
      _upload_isolate_step_data(api, 'test'),
      _upload_isolate_step_data(api, 'trigger'),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.step_data('dart', retcode=1),
      api.post_process(StepFailure, 'dart'),
      api.post_process(Filter('dart')),
  )

  chunk_upload_step = ('upload new results.'
                       'upload test results to big query.'
                       'upload results chunk to big query')
  yield api.test(
      'upload-error-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data(chunk_upload_step, retcode=1),
      api.post_process(StepException, chunk_upload_step),
      api.post_process(StatusException),
      api.post_process(Filter(chunk_upload_step)),
  )

  download_step = 'download previous results.gsutil download previous results'
  yield api.test(
      'download-results-error-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text("1234", name="latest")),
      api.step_data(download_step, retcode=1),
      api.post_process(StepException, download_step),
      api.post_process(StatusException),
      api.post_process(Filter(download_step)),
  )

  deflake_step = 'deflaking.list tests to deflake (co19)'
  yield api.test(
      'failed-to-get-tests-to-deflake-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19_sharded', git_repo='https://dart.googlesource.com/sdk'),
      _upload_isolate_step_data(api),
      api.step_data(deflake_step, retcode=1),
      api.post_process(StepException, deflake_step),
      api.post_process(StatusException),
      api.post_process(Filter(deflake_step)),
  )

  update_flaky_step = 'update flakiness information'
  yield api.test(
      'failed-to-update-flaky-data-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19_sharded', git_repo='https://dart.googlesource.com/sdk'),
      _upload_isolate_step_data(api),
      api.step_data(update_flaky_step, retcode=1),
      api.post_process(StepException, update_flaky_step),
      api.post_process(StatusException),
      api.post_process(Filter(update_flaky_step)),
  )

  update_latest_step = 'upload new results.gsutil update "latest" reference'
  yield api.test(
      'failed-to-update-latest-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19_sharded', git_repo='https://dart.googlesource.com/sdk'),
      _upload_isolate_step_data(api),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data(update_latest_step, retcode=1),
      api.post_process(StepException, update_latest_step),
      api.post_process(StatusException),
      api.post_process(Filter(update_latest_step)),
  )

  upload_step = 'upload new results.gsutil upload revision'
  yield api.test(
      'failed-to-upload-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data(upload_step, retcode=1),
      api.post_process(StepException, upload_step),
      api.post_process(StatusException),
      api.post_process(Filter(upload_step)),
  )

  publish_step = 'upload new results.publish results to pub/sub'
  yield api.test(
      'failed-to-publish-is-infra-failure',
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data(publish_step, retcode=1),
      api.post_process(StepException, publish_step),
      api.post_process(StatusException),
      api.post_process(Filter(publish_step)),
  )

  yield api.test(
      'fuzz-test',
      _upload_isolate_step_data(api),
      api.buildbucket.ci_build(
          builder='fuzz-linux',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data(
          'make a fuzz_shard_1',
          api.swarming.collect([
              api.swarming.task_result(
                  id='0', name='make a fuzz_shard_1', failure=True)
          ])),
      api.post_process(MustRun, 'make a fuzz_shard_2'),
      api.post_process(StepFailure, 'make a fuzz_shard_1'),
      api.post_process(StatusFailure),
  )

  yield api.test(
      'fuzz-shard-timeout',
      _upload_isolate_step_data(api),
      api.buildbucket.ci_build(
          builder='fuzz-linux',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data(
          'make a fuzz_shard_1',
          api.swarming.collect([
              api.swarming.task_result(
                  id='0',
                  name='make a fuzz_shard_1',
                  state=api.swarming.TaskState.TIMED_OUT)
          ])),
      api.post_process(MustRun, 'make a fuzz_shard_2'),
      api.post_process(StepException, 'make a fuzz_shard_1'),
      api.post_process(StatusException),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'test-shard-failure',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data(
          'test1_shard_1',
          api.swarming.collect([
              api.swarming.task_result(
                  id='0', name='test1_shard_1', failure=True)
          ])),
      _upload_isolate_step_data(api, 'test'),
      _upload_isolate_step_data(api, 'trigger'),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.post_process(StepException, 'test1_shard_1'),
      api.post_process(StatusException),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'local-test-shard-failure',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('test3_shard_2', retcode=1),
      _upload_isolate_step_data(api, 'test'),
      _upload_isolate_step_data(api, 'trigger'),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.post_process(StepException, 'test3_shard_2'),
      api.post_process(StatusException),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'co19',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19_sharded',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'co19', 1, False),
      _upload_isolate_step_data(api),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(StatusSuccess),
      api.post_process(Filter().include_re(r'.*co19.*')),
  )

  yield api.test(
      'co19-no-revinfo',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('gclient get package versions',
                    api.json.output(name='revinfo', data={})),
      api.post_process(StatusException),
      api.post_process(Filter().include_re(r'.*(get package versions|co19).*')),
  )

  yield api.test(
      'custom-test-runner-failure-is-infra-failure',
      api.properties(shard_timeout='600'),
      api.buildbucket.try_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox-try',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('custom_runner', retcode=1),
      _upload_isolate_step_data(api, 'test'),
      api.post_process(StepException, 'custom_runner'),
      api.post_process(StatusException),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'invalid-fileset-upload-output',
      api.buildbucket.ci_build(
          builder='co19_sharded',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('')),
      api.post_process(StatusException),
      api.post_process(Filter('upload testing fileset test')),
  )