recipe_modules/bisect_build/tests/tests.py

# Copyright (c) 2020, the Dart project authors. All rights reserved. Use of this
# source code is governed by a BSD-style license that can be found in the
# LICENSE file.

from recipe_engine.post_process import (
    DropExpectation,
    DoesNotRun,
    DoesNotRunRE,
    Filter,
    MustRun,
    StatusSuccess,
)

from recipe_engine.recipe_api import Property

DEPS = [
    'depot_tools/gitiles',
    'bisect_build',
    'recipe_engine/buildbucket',
    'recipe_engine/properties',
    'recipe_engine/step',
]

PROPERTIES = {
    'current_failure':
        Property(
            kind=str,
            help='Reason for current failure or "SUCCESS"',
            default='SUCCESS'),
    'is_experimental':
        Property(
            kind=bool,
            help='Controls whether the bisection build should be scheduled as'
            ' experimental',
            default=False),
}


def RunSteps(api, current_failure, is_experimental):
  api.bisect_build.schedule('https://dummy/repo/url', current_failure,
                            is_experimental)


def _test(api, name, failure=None, is_experimental=False, build_number=3):
  data = api.test(
      name,
      api.buildbucket.ci_build(
          builder='builder',
          build_number=build_number,
          git_repo='https://dart.googlesource.com/sdk',
          revision='f' * 8))
  if failure:
    data += api.properties(
        current_failure=failure, is_experimental=is_experimental)
  return data


# TODO(karlklose): remove is_experimental support once we know that we do not
# want to use it again.
def _sets_experimental_flag(check, step_odict, step):
  # Verify that the step contains the option for an experimental build
  input = str(step_odict[step].stdin)
  check('"experimental": "YES"' in input)


def GenTests(api):

  yield _test(api, 'basic')

  BUILDER = 'foo'
  GIT_HASH = '2d72510e447ab60a9728aeea2362d8be2cbd7789'

  STARTS_BISECTION = api.post_process(
      MustRun, 'schedule bisect (f4d35da881f8fd329a4d3e01dd78b66a502d5c49)')
  DOES_NOT_BISECT = api.post_process(DoesNotRunRE, r'schedule bisect.*')

  builds = [api.bisect_build.build(api, BUILDER, 2, GIT_HASH)]
  yield (_test(api, 'starts bisection', failure='failure') + api.step_data(
      'gitiles log: 2d72510e447ab60a9728aeea2362d8be2cbd7789..ffffffff',
      api.gitiles.make_log_test_data('master')) +
         api.bisect_build.fetch_previous_builds(api, builds) + STARTS_BISECTION)

  yield (_test(
      api,
      'start-experimental-bisection',
      failure='failure',
      is_experimental=True) + api.step_data(
          'gitiles log: 2d72510e447ab60a9728aeea2362d8be2cbd7789..ffffffff',
          api.gitiles.make_log_test_data('master')) +
         api.bisect_build.fetch_previous_builds(api, builds) + api.post_process(
             MustRun,
             'schedule bisect (f4d35da881f8fd329a4d3e01dd78b66a502d5c49)') +
         api.post_process(StatusSuccess) + api.post_process(
             _sets_experimental_flag,
             'schedule bisect (f4d35da881f8fd329a4d3e01dd78b66a502d5c49)'))

  builds = [api.bisect_build.build(api, BUILDER, 2, GIT_HASH, is_success=False)]
  yield (_test(
      api, 'do-not-start-bisect-if-previous-build-failed', failure='failure') +
         api.bisect_build.fetch_previous_builds(api, builds) + DOES_NOT_BISECT)

  builds = [
      api.bisect_build.build(api, BUILDER, 2, GIT_HASH, is_bisection=True),
      api.bisect_build.build(api, BUILDER, 1, GIT_HASH, is_bisection=True),
      api.bisect_build.build(api, BUILDER, 0, GIT_HASH, is_bisection=True),
  ]
  yield (_test(
      api,
      'do-not-start-bisect-if-all-previous-builds-are-non-regular',
      failure='failure') + api.bisect_build.fetch_previous_builds(api, builds) +
         DOES_NOT_BISECT)

  yield (_test(
      api, 'do-not-start-bisect-without-previous-build', failure='failure') +
         DOES_NOT_BISECT)

  yield api.test(
      'do-not-start-bisect-without-current-revision',
      api.buildbucket.ci_build(
          builder='builder',
          git_repo='https://dart.googlesource.com/sdk',
          revision=None), api.properties(current_failure='failure'),
      DOES_NOT_BISECT)

  yield (_test(api, 'continue-bisect-on-failure', failure='failure') +
         api.properties(
             bisect_newer=['a', 'b', 'c'],
             bisect_older=['c', 'd', 'e'],
             bisect_base_build=4711,
             bisect_reason='failure') +
         api.post_process(MustRun, 'schedule bisect (d)') +
         api.post_process(Filter('schedule bisect (d)')))

  yield (
      _test(api, 'continue-bisect-on-failure-experimental', failure='failure') +
      api.properties(
          bisect_newer=['a', 'b', 'c'],
          bisect_older=['c', 'd', 'e'],
          bisect_base_build=4711,
          bisect_reason='failure',
          is_experimental=True) +
      api.post_process(MustRun, 'schedule bisect (d)') +
      api.post_process(_sets_experimental_flag, 'schedule bisect (d)') +
      api.post_process(Filter('schedule bisect (d)')))

  yield (_test(api, 'continue-bisect-on-success') + api.properties(
      bisect_newer=['a', 'b', 'c'],
      bisect_older=['c', 'd', 'e'],
      bisect_base_build=4711,
      bisect_reason='failure') +
         api.post_process(MustRun, 'schedule bisect (b)') +
         api.post_process(DoesNotRun, 'schedule bisect (d)') +
         api.post_process(Filter().include_re(r'schedule bisect \(.*\)')))

  yield (_test(api, 'fan-out-on-distinct-failure',
               failure='different failure') + api.properties(
                   bisect_newer=['a', 'b', 'c'],
                   bisect_older=['c', 'd', 'e'],
                   bisect_base_build=4711,
                   bisect_reason='failure') +
         api.post_process(MustRun, 'schedule bisect (b)') +
         api.post_process(MustRun, 'schedule bisect (d)') +
         api.post_process(Filter().include_re(r'schedule bisect \(.*\)')))

  yield (_test(api, 'stop-bisect') + api.properties(
      bisect_newer=[],
      bisect_older=[],
      bisect_base_build=4711,
      bisect_reason='failure') + DOES_NOT_BISECT +
         api.post_process(DropExpectation))