# Copyright 2018 The Chromium Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.


from recipe_engine.post_process import (
    DoesNotRun,
    DoesNotRunRE,
    DropExpectation,
    Filter,
    MustRun,
    MustRunRE,
    StatusException,
    StatusFailure,
    StatusSuccess,
    StepCommandContains,
    StepException,
    StepFailure,
)

DEPS = [
    'dart',
    'depot_tools/tryserver',
    'recipe_engine/buildbucket',
    'recipe_engine/file',
    'recipe_engine/json',
    'recipe_engine/properties',
    'recipe_engine/platform',
    'recipe_engine/raw_io',
    'recipe_engine/runtime',
    'recipe_engine/step',
    'recipe_engine/swarming',
]


CANNED_OUTPUT_DIR = {
    'logs.json':
        '{"test":"test1", "configuration": "example_configuration"}\n',
    'results.json':
        '{"name":"test1", "configuration": example_configuration"}\n'
        '{"name":"test2", "configuration": example_configuration_2"}\n',
}

CANNED_FLAKY_OUTPUT_DIR = {
    'logs.json':
        '{"name": "Flaky/Test/1", "configuration": "example_configuration"}\n'
        '{"name": "Flaky/Test/2", "configuration": "example_configuration"}\n',
    'results.json':
        '{"name":"Flaky/Test/1", "configuration": "example_configuration"}\n'
        '{"name":"Flaky/Test/2", "configuration": "example_configuration"}\n',
}


TRIGGER_RESULT = {
  "results": [
    {
      "build": {
        "id": "8963024236039183808",
        "tags": [
          "builder:analyzer-linux-release",
          "parent_buildername:dart-sdk-linux"
        ],
        "url": "https://ci.chromium.org/p/dart/builds/b8963024236039183808",
      }
    },
    {
      "build": {
        "id": "8963024236039836208",
        "tags": [
          "builder:analyzer-strong-linux-release",
          "parent_buildername:dart-sdk-linux"
        ],
        "url": "https://ci.chromium.org/p/dart/builds/b8963024236039836208",
      }
    },
    {
      "build": {
        "id": "8963024236039228096",
        "tags": [
          "builder:analyzer-analysis-server-linux",
          "parent_buildername:dart-sdk-linux"
        ],
        "url": "https://ci.chromium.org/p/dart/builds/b8963024236039228096",
      }
    }
  ]
}


TEST_MATRIX = {
    "filesets": {
        "test": "[]",
        "trigger": "[]"
    },
    "global": {
        "chrome": "66.0.3359.139",
        "firefox": "61"
    },
    "builder_configurations": [
        {
            "builders": ["vm-kernel-win-release-simarm_x64",],
            "meta": {},
            "steps": [{
                "name":
                    "gn",
                "script":
                    "tools/gn.py",
                "arguments": [
                    "--bytecode", "--gen-snapshot=${build_root}/gen_snapshot"
                ]
            },
                      {
                          "name": "build",
                          "script": "tools/build.py",
                          "arguments": ["--arch=x64", "runtime"]
                      },
                      {
                          "name":
                              "test1",
                          "script":
                              "tools/test.py",
                          "arguments": [
                              "foo", "-ndartk-${system}-${mode}-${arch}",
                              "language_2", "co19_2/sel", "--exclude_suite=co19"
                          ],
                          "shards":
                              2,
                          "fileset":
                              "test"
                      },
                      {
                          "name":
                              "test2",
                          "arguments": [
                              "foo", "--bar",
                              "-ndartk-${system}-${mode}-${arch}"
                          ],
                      },
                      {
                          "name": "trigger",
                          "fileset": "trigger",
                          "trigger": ["foo-builder", "bar-builder"]
                      },
                      {
                          "name": "dart",
                          "script": "out/ReleaseX64/dart",
                          "arguments": ["--bar", "foo.dart"]
                      },
                      {
                          "name":
                              "test3",
                          "arguments": [
                              "-ndartk-${system}-${mode}-${arch}", "foo",
                              "--bar"
                          ],
                          "fileset":
                              "test",
                          "shards":
                              2
                      }]
        },
        {
            "builders": ["co19",],
            "meta": {},
            "steps": [{
                "name": "co19",
                "arguments": ["co19/nnbd", "co19_2"],
                "fileset": "test",
                "shards": 1
            }]
        },
        {
            "builders": ["analyzer-linux-release",],
            "meta": {},
            "steps": [{
                "name": "build",
                "script": "tools/build.py"
            },
                      {
                          "name": "test1",
                          "script": "tools/test.py",
                          "fileset": "test",
                          "shards": 2,
                          "arguments": ["-nunittest-asserts-${mode}-${system}"]
                      },
                      {
                          "name":
                              "test2",
                          "arguments": [
                              "foo", "--bar",
                              "-ndartk-${system}-${mode}-${arch}"
                          ],
                      },
                      {
                          "name": "trigger",
                          "fileset": "trigger",
                          "trigger": ["foo-builder", "bar-builder"]
                      },
                      {
                          "name":
                              "test3",
                          "arguments": [
                              "-nanalyzer-asserts-${system}", "foo", "--bar"
                          ],
                          "fileset":
                              "test",
                          "shards":
                              2
                      }]
        },
        {
            "builders": [
                "dart2js-strong-mac-x64-chrome",
                "dart2js-strong-linux-x64-firefox",
                "dart2js-strong-win-x64-chrome"
            ],
            "meta": {},
            "steps": [{
                "name":
                    "test1",
                "script":
                    "tools/test.py",
                "arguments": [
                    "-ndart2js-${system}-${runtime}", "foo", "--bar",
                    "-e co19, language_2"
                ],
                "shards":
                    2,
                "fileset":
                    "test"
            },
                      {
                          "name": "custom",
                          "script": "tools/custom_thing.py",
                          "arguments": ["foo", "--bar", "--buildername"]
                      },
                      {
                          "name": "custom_runner",
                          "script": "tools/custom_test_runner.py",
                          "testRunner": True,
                          "arguments": ["foo", "--bar", "--buildername"]
                      },
                      {
                          "name":
                              "test2",
                          "arguments": [
                              "-ndart2js-${system}-${runtime}", "foo", "--bar",
                              "co19"
                          ],
                      }]
        },
        {
            "builders": ["vm-kernel-mac-release-x64"],
            "meta": {},
            "steps": [{
                "name": "build",
                "script": "tools/build.py",
                "arguments": []
            },
                      {
                          "name": "custom",
                          "script": "out/custom_thing",
                          "arguments": ["foo", "--bar", "--buildername"]
                      }]
        },
        {
            "builders": ["vm-kernel-precomp-android-release-armsimdbc64"],
            "meta": {},
            "steps": [{
                "name": "android",
                "shards": 2,
                "fileset": "test",
                "arguments": ["-ndartkp-android-${mode}-${arch}"]
            }]
        },
        {
            "builders": ["fuzz-linux"],
            "steps": [{
                "name":
                    "make a fuzz",
                "script":
                    "out/ReleaseX64/dart",
                "arguments": [
                    "runtime/tools/dartfuzz/dartfuzz_test.dart", "--isolates",
                    "8", "--no-show-stats", "--time", "2700"
                ],
                "shards":
                    3,
                "fileset":
                    "test"
            }]
        }
    ]
}

RESULT_DATA = (
    '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t01"'
    ',"configuration":"dartk-linux-product-x64","suite":"co19_2",'
    '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t01",'
    '"time_ms":451,"result":"CompileTimeError","expected":"CompileTimeError",'
    '"matches":true,"commit_time":1551185312,'
    '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",'
    '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,'
    '"previous_flaky":false,"previous_result":"CompileTimeError",'
    '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"previous_commit_time":1551185312,"previous_build_number":2403,'
    '"changed":false}\n' +
    '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t02"'
    ',"configuration":"dartk-linux-product-x64","suite":"co19_2",'
    '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t02",'
    '"time_ms":496,"result":"CompileTimeError","expected":"CompileTimeError",'
    '"matches":true,"commit_time":1551185312,'
    '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",'
    '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,'
    '"previous_flaky":false,"previous_result":"CompileTimeError",'
    '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"previous_commit_time":1551185312,"previous_build_number":2403,'
    '"changed":false}\n')

RESULT_DATA_MULTIPLE_CONFIGURATIONS = (
    '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t01"'
    ',"configuration":"dartk-linux-product-x64","suite":"co19_2",'
    '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t01",'
    '"time_ms":451,"result":"CompileTimeError","expected":"CompileTimeError",'
    '"matches":true,"commit_time":1551185312,'
    '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",'
    '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,'
    '"previous_flaky":false,"previous_result":"CompileTimeError",'
    '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"previous_commit_time":1551185312,"previous_build_number":2403,'
    '"changed":false}\n' +
    '{"name":"co19_2/Language/Classes/Abstract_Instance_Members/inherited_t02"'
    ',"configuration":"dartk-linux-release-x64","suite":"co19_2",'
    '"test_name":"Language/Classes/Abstract_Instance_Members/inherited_t02",'
    '"time_ms":496,"result":"CompileTimeError","expected":"CompileTimeError",'
    '"matches":true,"commit_time":1551185312,'
    '"commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"build_number":"2404","builder_name":"vm-kernel-linux-product-x64",'
    '"bot_name":"trusty-dart-68765ebb-us-central1-b-2ls0","flaky":false,'
    '"previous_flaky":false,"previous_result":"CompileTimeError",'
    '"previous_commit_hash":"f0042a32250a8a6193e6d07e2b6508b13f43c864",'
    '"previous_commit_time":1551185312,"previous_build_number":2403,'
    '"changed":false}\n')


def RunSteps(api):
  api.dart.checkout('clobber' in api.properties,
                    api.properties.get('custom_vars', {}))

  build_args = ['--super-fast']
  api.dart.build(build_args, name='can_time_out')

  api.dart.kill_tasks()
  api.dart.read_debug_log()

  api.dart.test(test_data=TEST_MATRIX)


def _canned_step(api, name, shards=0, local_shard=True, deflake=True):
  step_name = '%s_shard_%s' % (name, 1) if shards > 0 else name
  data = None
  for i in range(1 if shards > 0 else 0, shards + 1):
    if not data:
      data = _canned_output_dir(api, step_name)
    else:
      data += _canned_output_dir(api, '%s_shard_%s' % (name, i))
    if deflake:
      deflaking_name = name if local_shard else '%s_shard_1' % name
      data += _canned_output_dir(
          api, deflaking_name, CANNED_FLAKY_OUTPUT_DIR, prefix='deflaking.')
  if deflake:
    data += api.step_data('deflaking.list tests to deflake (%s)' % name,
                stdout=api.raw_io.output('Flaky/Test/1\nFlaky/Test/2'))

  return data


def _canned_output_dir(api, step_name, content=None, prefix=''):
  content = content or CANNED_OUTPUT_DIR
  data = api.empty_test_data()
  for filename in ('logs.json', 'results.json'):
    data += api.step_data('%sread %s for %s' % (prefix, filename, step_name),
                          api.file.read_text(text_content=content[filename]))
  return data


def GenTests(api):
  yield api.test(
      'basic',
      api.properties(
          shard_timeout='600', custom_vars={'download_firefox': True}),
      api.buildbucket.try_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox-try',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'custom_runner', deflake=False),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.tryserver.gerrit_change_target_ref('refs/heads/stable'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text('123', name='latest')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(DoesNotRun, "update blamelists"),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'branch-builder-does-not-publish-results',
      api.properties(shard_timeout='600'),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart',
          git_ref="refs/heads/some-branch"),
      _canned_step(api, 'co19'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text('123', name='latest')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(MustRunRE, '.*.gsutil upload results.json'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       ['tools/bots/compare_results.dart']),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'ci-test-data-branch-builder-does-publish-results',
      api.properties(shard_timeout='600'),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart',
          git_ref="refs/heads/ci-test-data"),
      _canned_step(api, 'co19'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text('123', name='latest')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(MustRunRE, '.*.gsutil upload results.json'),
      api.post_process(MustRunRE, '.*publish results to pub/sub'),
      api.post_process(
          StepCommandContains, 'test results',
          ['[CACHE]/builder/sdk/tools/bots/get_builder_status.dart']),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'release-builder-does-not-publish-results',
      api.properties(shard_timeout='600'),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='co19-dev',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart',
          git_ref="refs/heads/dev"),
      _canned_step(api, 'co19'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text('123', name='latest')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(MustRunRE, '.*.gsutil upload results.json'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       ['tools/bots/compare_results.dart']),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'bisection-does-not-refer-to-latest-build-or-publish-results',
      api.properties(
          shard_timeout='600', bisect_reason="FAILURE", bisect_base_build=4711),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'custom_runner', deflake=False),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(DoesNotRunRE, '.*find latest build'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       ['tools/bots/compare_results.dart']),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'bisection-on-test-branch-updates-blamelist',
      api.properties(
          shard_timeout='600', bisect_reason="FAILURE", bisect_base_build=4711),
      api.buildbucket.ci_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart',
          git_ref="refs/heads/ci-test-data"),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'custom_runner', deflake=False),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data(
          'add fields to result records',
          api.raw_io.output_text(RESULT_DATA_MULTIPLE_CONFIGURATIONS)),
      api.post_process(DoesNotRunRE, '.*find latest build'),
      api.post_process(DoesNotRunRE, '.*publish results to pub/sub'),
      api.post_process(StepCommandContains, 'test results',
                       ['tools/bots/compare_results.dart']),
      api.post_process(
          MustRun, 'update blamelists for configuration '
          '"dartk-linux-product-x64"'),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'analyzer-linux-release',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'test3', 2),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data(
          'upload testing fileset trigger',
          stdout=api.raw_io.output('trigger_hash')),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'experimental-builds-are-side-effect-free',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'test1', 2, False),
      _canned_step(api, 'test2'),
      _canned_step(api, 'test3', 2),
      api.runtime(is_luci=True, is_experimental=True),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data(
          'upload testing fileset trigger',
          stdout=api.raw_io.output('trigger_hash')),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(DoesNotRunRE, r'.*publish results.*'),
      api.post_process(StatusSuccess),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'build-failure-in-matrix',
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('build', retcode=1),
      api.post_process(DoesNotRun, 'test1'),
      api.post_process(StepFailure, 'build'),
      api.post_process(StatusFailure),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'basic-missing-name',
      api.buildbucket.ci_build(
          builder='this-name-does-not-exist-in-test-matrix',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.post_process(StatusException),
  )

  yield api.test(
      'basic-timeout',
      api.buildbucket.ci_build(
          builder='times-out',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('can_time_out', times_out_after=60 * 61 + 1),
      api.post_process(StatusFailure),
  )

  yield api.test(
      'basic-failure',
      api.buildbucket.ci_build(
          builder='build-fail',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('can_time_out', retcode=1),
      api.post_process(StatusFailure),
  )

  yield api.test(
      'failed-tests',
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'co19', 1, False),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data('test results', retcode=1),
      api.post_process(StepFailure, 'test results'),
      api.post_process(StatusFailure),
      api.post_process(Filter('test results')),
  )

  yield api.test(
      'failed-to-get-test-results',
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'co19', 1, False),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data('test results', retcode=2),
      api.post_process(StepException, 'test results'),
      api.post_process(StatusException),
      api.post_process(Filter('test results')),
  )

  yield api.test(
      'vm-win',
      api.platform('win', 64),
      api.properties(bot_id='win-dart-123'),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='vm-kernel-win-release-simarm_x64',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data(
          'upload testing fileset trigger',
          stdout=api.raw_io.output('trigger_hash')),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'basic-mac',
      api.platform('mac', 64),
      api.properties(bot_id='mac-dart-123'),
      api.buildbucket.ci_build(
          revision='a' * 40,
          build_number=1357,
          builder='dart2js-strong-mac-x64-chrome-dev',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.properties(clobber='True', custom_vars={'download_chrome': True}),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'example-mac',
      api.platform('mac', 64),
      api.buildbucket.ci_build(
          builder='vm-kernel-mac-release-x64',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'example-android',
      api.platform('linux', 64),
      api.buildbucket.ci_build(
          builder='vm-kernel-precomp-android-release-armsimdbc64',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'android', 2, False),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.post_process(StatusSuccess),
  )

  yield api.test(
      'non-test-step-error-is-step-failure',
      api.buildbucket.ci_build(
          builder='vm-kernel-win-release-simarm_x64',
          git_repo='https://dart.googlesource.com/sdk'),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.step_data('dart', retcode=1),
      api.post_process(StepFailure, 'dart'),
      api.post_process(Filter('dart')),
  )

  chunk_upload_step = ('upload new results.'
                       'upload test results to big query.'
                       'upload results chunk to big query')
  yield api.test(
      'upload-error-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data(chunk_upload_step, retcode=1),
      api.post_process(StepException, chunk_upload_step),
      api.post_process(StatusException),
      api.post_process(Filter(chunk_upload_step)),
  )

  download_step = 'download previous results.gsutil download previous results'
  yield api.test(
      'download-results-error-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data('download previous results.gsutil find latest build',
                    api.raw_io.output_text("1234", name="latest")),
      api.step_data(download_step, retcode=1),
      api.post_process(StepException, download_step),
      api.post_process(StatusException),
      api.post_process(Filter(download_step)),
  )

  deflake_step = 'deflaking.list tests to deflake (co19)'
  yield api.test(
      'failed-to-get-tests-to-deflake-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data(deflake_step, retcode=1),
      api.post_process(StepException, deflake_step),
      api.post_process(StatusException),
      api.post_process(Filter(deflake_step)),
  )

  update_flaky_step = 'update flakiness information'
  yield api.test(
      'failed-to-update-flaky-data-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data(update_flaky_step, retcode=1),
      api.post_process(StepException, update_flaky_step),
      api.post_process(StatusException),
      api.post_process(Filter(update_flaky_step)),
  )

  update_latest_step = 'upload new results.gsutil update "latest" reference'
  yield api.test(
      'failed-to-update-latest-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data(update_latest_step, retcode=1),
      api.post_process(StepException, update_latest_step),
      api.post_process(StatusException),
      api.post_process(Filter(update_latest_step)),
  )

  upload_step = 'upload new results.gsutil upload revision'
  yield api.test(
      'failed-to-upload-is-infra-failure',
      api.buildbucket.ci_build(
          builder='co19', git_repo='https://dart.googlesource.com/sdk'),
      api.step_data(upload_step, retcode=1),
      api.post_process(StepException, upload_step),
      api.post_process(StatusException),
      api.post_process(Filter(upload_step)),
  )

  publish_step = 'upload new results.publish results to pub/sub'
  yield api.test(
      'failed-to-publish-is-infra-failure',
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.step_data(publish_step, retcode=1),
      api.post_process(StepException, publish_step),
      api.post_process(StatusException),
      api.post_process(Filter(publish_step)),
  )

  yield api.test(
      'fuzz-test',
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.buildbucket.ci_build(
          builder='fuzz-linux',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data(
          'make a fuzz_shard_1',
          api.swarming.collect([
              api.swarming.task_result(
                  id='0', name='make a fuzz_shard_1', failure=True)
          ])),
      api.post_process(MustRun, 'make a fuzz_shard_2'),
      api.post_process(StepFailure, 'make a fuzz_shard_1'),
      api.post_process(StatusFailure),
  )

  yield api.test(
      'fuzz-shard-timeout',
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.buildbucket.ci_build(
          builder='fuzz-linux',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data(
          'make a fuzz_shard_1',
          api.swarming.collect([
              api.swarming.task_result(
                  id='0',
                  name='make a fuzz_shard_1',
                  state=api.swarming.TaskState.TIMED_OUT)
          ])),
      api.post_process(MustRun, 'make a fuzz_shard_2'),
      api.post_process(StepException, 'make a fuzz_shard_1'),
      api.post_process(StatusException),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'test-shard-failure',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data(
          'test1_shard_1',
          api.swarming.collect([
              api.swarming.task_result(
                  id='0', name='test1_shard_1', failure=True)
          ])),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data(
          'upload testing fileset trigger',
          stdout=api.raw_io.output('trigger_hash')),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.post_process(StepException, 'test1_shard_1'),
      api.post_process(StatusException),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'local-test-shard-failure',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          builder='analyzer-linux-release',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('test3_shard_2', retcode=1),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data(
          'upload testing fileset trigger',
          stdout=api.raw_io.output('trigger_hash')),
      api.step_data('buildbucket.put', stdout=api.json.output(TRIGGER_RESULT)),
      api.post_process(StepException, 'test3_shard_2'),
      api.post_process(StatusException),
      api.post_process(DropExpectation),
  )

  yield api.test(
      'co19',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      _canned_step(api, 'co19', 1, False),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.step_data('add fields to result records',
                    api.raw_io.output_text(RESULT_DATA)),
      api.post_process(StatusSuccess),
      api.post_process(Filter().include_re(r'.*co19.*')),
  )

  yield api.test(
      'co19-no-revinfo',
      api.properties(bot_id='trusty-dart-123'),
      api.buildbucket.ci_build(
          revision='3456abce78ef',
          build_number=1357,
          builder='co19',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('gclient get package versions',
                    api.json.output(name='revinfo', data={})),
      api.post_process(StatusException),
      api.post_process(Filter().include_re(r'.*(get package versions|co19).*')),
  )

  yield api.test(
      'custom-test-runner-failure-is-infra-failure',
      api.properties(shard_timeout='600'),
      api.buildbucket.try_build(
          build_number=1357,
          builder='dart2js-strong-linux-x64-firefox-try',
          git_repo='https://dart.googlesource.com/sdk',
          project='dart'),
      api.step_data('custom_runner', retcode=1),
      api.step_data(
          'upload testing fileset test', stdout=api.raw_io.output('test_hash')),
      api.post_process(StepException, 'custom_runner'),
      api.post_process(StatusException),
      api.post_process(DropExpectation),
  )