recipe_modules/bisect_build/tests/tests.py

# Copyright (c) 2020, the Dart project authors. All rights reserved. Use of this
# source code is governed by a BSD-style license that can be found in the
# LICENSE file.

from recipe_engine.post_process import DropExpectation, DoesNotRun, DoesNotRunRE, Filter, MustRun, StatusSuccess

from recipe_engine.recipe_api import Property

DEPS = [
    'depot_tools/gitiles',
    'bisect_build',
    'recipe_engine/properties',
    'recipe_engine/step',
    'recipe_engine/buildbucket',
]

PROPERTIES = {
    'current_failure':
        Property(
            kind=str,
            help="Reason for current failure or 'SUCCESS'",
            default="SUCCESS"),
    'is_experimental':
        Property(
            kind=bool,
            help="Controls whether the bisection build should be scheduled as experimental",
            default=False),
}


def RunSteps(api, current_failure, is_experimental):
  api.bisect_build.schedule("https://dummy/repo/url", current_failure,
                            is_experimental)


def _test(api, name, failure=None, is_experimental=False):
  data = api.test(
      name,
      api.buildbucket.ci_build(
          builder='builder',
          git_repo='https://dart.googlesource.com/sdk',
          revision='f' * 8))
  if failure:
    data += api.properties(
        current_failure=failure, is_experimental=is_experimental)
  return data


def _sets_experimental_flag(check, step_odict, step):
  # Verify that the step contains the option for an experimental build
  input = str(step_odict[step].stdin)
  check('"experimental": "YES"' in input)


def GenTests(api):
  yield _test(api, 'basic')

  yield (_test(api, 'starts bisection', failure='failure') + api.step_data(
      'gitiles log: 2d72510e447ab60a9728aeea2362d8be2cbd7789..ffffffff',
      api.gitiles.make_log_test_data('master')) +
         api.buildbucket.simulated_search_results(
             [
                 api.buildbucket.ci_build_message(status='SUCCESS'),
             ],
             step_name='fetch previous build') + api.post_process(
                 MustRun,
                 'schedule bisect (f4d35da881f8fd329a4d3e01dd78b66a502d5c49)'))

  yield (_test(
      api,
      'start-experimental-bisection',
      failure='failure',
      is_experimental=True) + api.step_data(
          'gitiles log: 2d72510e447ab60a9728aeea2362d8be2cbd7789..ffffffff',
          api.gitiles.make_log_test_data('master')) +
         api.buildbucket.simulated_search_results(
             [
                 api.buildbucket.ci_build_message(status='SUCCESS'),
             ],
             step_name='fetch previous build') + api.post_process(
                 MustRun,
                 'schedule bisect (f4d35da881f8fd329a4d3e01dd78b66a502d5c49)') +
         api.post_process(StatusSuccess) + api.post_process(
             _sets_experimental_flag,
             'schedule bisect (f4d35da881f8fd329a4d3e01dd78b66a502d5c49)'))

  yield (_test(
      api, 'do-not-start-bisect-if-previous-build-failed', failure="failure") +
         api.buildbucket.simulated_search_results(
             [
                 api.buildbucket.ci_build_message(status='FAILURE'),
             ],
             step_name='fetch previous build') + api.post_process(
                 DoesNotRunRE, r'schedule bisect.*'))

  yield (_test(
      api, 'do-not-start-bisect-without-previous-build', failure="failure") +
         api.buildbucket.simulated_search_results(
             [], step_name='fetch previous build') + api.post_process(
                 DoesNotRunRE, r'schedule bisect.*'))

  yield api.test(
      'do-not-start-bisect-without-current-revision',
      api.buildbucket.ci_build(
          builder='builder',
          git_repo='https://dart.googlesource.com/sdk',
          revision=None), api.properties(current_failure="failure"),
      api.post_process(DoesNotRunRE, r'schedule bisect.*'))

  yield (_test(api, 'continue-bisect-on-failure', failure="failure") +
         api.properties(
             bisect_newer=['a', 'b', 'c'],
             bisect_older=['c', 'd', 'e'],
             bisect_base_build=4711,
             bisect_reason="failure") +
         api.post_process(MustRun, 'schedule bisect (d)') +
         api.post_process(Filter('schedule bisect (d)')))

  yield (
      _test(api, 'continue-bisect-on-failure-experimental', failure="failure") +
      api.properties(
          bisect_newer=['a', 'b', 'c'],
          bisect_older=['c', 'd', 'e'],
          bisect_base_build=4711,
          bisect_reason="failure",
          is_experimental=True) +
      api.post_process(MustRun, 'schedule bisect (d)') +
      api.post_process(_sets_experimental_flag, 'schedule bisect (d)') +
      api.post_process(Filter('schedule bisect (d)')))

  yield (_test(api, 'continue-bisect-on-success') + api.properties(
      bisect_newer=['a', 'b', 'c'],
      bisect_older=['c', 'd', 'e'],
      bisect_base_build=4711,
      bisect_reason="failure") +
         api.post_process(MustRun, 'schedule bisect (b)') +
         api.post_process(DoesNotRun, 'schedule bisect (d)') +
         api.post_process(Filter().include_re(r'schedule bisect \(.*\)')))

  yield (_test(api, 'fan-out-on-distinct-failure',
               failure="different failure") + api.properties(
                   bisect_newer=['a', 'b', 'c'],
                   bisect_older=['c', 'd', 'e'],
                   bisect_base_build=4711,
                   bisect_reason='failure') +
         api.post_process(MustRun, 'schedule bisect (b)') +
         api.post_process(MustRun, 'schedule bisect (d)') +
         api.post_process(Filter().include_re(r'schedule bisect \(.*\)')))

  yield (_test(api, 'stop-bisect') + api.properties(
      bisect_newer=[],
      bisect_older=[],
      bisect_base_build=4711,
      bisect_reason='failure') +
         api.post_process(DoesNotRunRE, r'schedule bisect.*') +
         api.post_process(DropExpectation))