| # Copyright 2020 The Dart Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| """A recipe that processes go/dart-cbuild results. |
| |
| The cbuild uploads files to a GCS bucket (one per SDK commit). This recipe reads |
| those files and starts a synthethic build of the "google" builder with the same |
| outcome as the cbuild and a link to the cbuild results. The result is deleted |
| from the GCS bucket so that it won't be processed again by the next run. |
| """ |
| |
| import re |
| |
| from google.protobuf import json_format |
| from recipe_engine import post_process |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2 |
| from PB.recipes.dart.dart.external_build import ExternalBuild |
| |
| DEPS = [ |
| 'depot_tools/gitiles', |
| 'depot_tools/gsutil', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/json', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/runtime', |
| 'recipe_engine/step', |
| ] |
| BUILDER = 'google' |
| BUCKET = 'ci.sandbox' |
| PROJECT = 'dart' |
| CBUILD_BUCKET_URL = 'gs://dart-cbuild-test-results' |
| CBUILD_RESULT_URL = 'https://goto.google.com/dart-cbuild/find/%s' |
| HOST = 'dart.googlesource.com' |
| REF = 'refs/heads/main' |
| REPO = 'sdk' |
| REPO_URL = 'https://%s/%s' % (HOST, REPO) |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| |
| def RunSteps(api): |
| results = _find_cbuild_results(api) |
| requests = [] |
| if not results: |
| # Nothing to do |
| return |
| for result in results: |
| is_presubmit = result['presubmit'] |
| branch = result['branch'] |
| if not is_presubmit and branch == 'main': |
| requests.append(_create_schedule_build_request(api, result)) |
| api.buildbucket.schedule(requests) |
| for result in results: |
| commit_hash = result['commit_hash'] |
| if not api.runtime.is_experimental: |
| api.gsutil.remove_url( |
| result['result_url'], name='delete result for %s' % commit_hash) |
| |
| |
| def _find_cbuild_results(api): |
| listing = api.gsutil.list( |
| CBUILD_BUCKET_URL, |
| name='find cbuild results', |
| stdout=api.raw_io.output_text()) |
| |
| results = [] |
| result_urls = listing.stdout.splitlines()[:50] # limit to 50 results per run |
| for result_url in result_urls: |
| if result_url.endswith('/'): |
| # ignore subdirectories |
| continue |
| _, commit_hash = result_url.rsplit('/', 1) |
| download = api.gsutil.download_url( |
| result_url, |
| api.json.output(name='result'), |
| name='download cbuild result for %s' % commit_hash) |
| cbuild_result_json = download.json.outputs['result'] |
| cbuild_success = cbuild_result_json['result'] |
| cbuild_regression = cbuild_result_json.get('regression', False) |
| cbuild_presubmit = cbuild_result_json.get('presubmit', False) |
| cbuild_branch = cbuild_result_json.get('branch', None) |
| cbuild_result = common_pb2.SUCCESS if cbuild_success else common_pb2.FAILURE |
| |
| assert not (cbuild_success and cbuild_regression) |
| |
| results.append({ |
| 'result_url': result_url, |
| 'commit_hash': commit_hash, |
| 'cbuild_result': cbuild_result, |
| 'cbuild_url': CBUILD_RESULT_URL % commit_hash, |
| 'presubmit': cbuild_presubmit, |
| 'branch': cbuild_branch, |
| }) |
| return results |
| |
| |
| def _create_schedule_build_request(api, result): |
| return api.buildbucket.schedule_request( |
| builder=BUILDER, |
| bucket=BUCKET, |
| project=PROJECT, |
| priority=20, |
| properties=json_format.MessageToDict( |
| ExternalBuild( |
| result=result['cbuild_result'], url=result['cbuild_url'])), |
| gitiles_commit=common_pb2.GitilesCommit( |
| host=HOST, |
| project=REPO, |
| ref=REF, |
| id=result['commit_hash'], |
| ), |
| inherit_buildsets=False, |
| ) |
| |
| |
| def GenTests(api): |
| |
| def schedules_build(build, scheduled): |
| |
| def _check(check, step_odict, step): |
| input = str(step_odict[step].stdin) |
| check(('"url": "https://goto.google.com/dart-cbuild/find/%s"' % |
| build in input) == scheduled) |
| |
| return api.post_process(_check, 'buildbucket.schedule') |
| |
| yield api.test( |
| 'no-results', |
| api.post_process(post_process.DoesNotRunRE, |
| '.*(delete|download|schedule).*'), |
| api.post_process(post_process.StatusSuccess), |
| api.post_process(post_process.DropExpectation), |
| ) |
| |
| def _cbuild_results(commit_hash, |
| result=True, |
| regression=False, |
| presubmit=False, |
| branch='main'): |
| cbuild_result = { |
| 'result': result, |
| 'regression': regression, |
| 'presubmit': presubmit, |
| } |
| if branch: |
| cbuild_result['branch'] = branch |
| return sum([ |
| api.step_data('gsutil download cbuild result for %s' % commit_hash, |
| api.json.output(cbuild_result, name='result')), |
| ], api.empty_test_data()) |
| |
| RESULT_SUCCESS = _cbuild_results('hash-of-success', result=True) |
| RESULT_REGRESSION = _cbuild_results( |
| 'hash-of-regression', result=False, regression=True) |
| RESULT_PRESUBMIT = _cbuild_results( |
| 'hash-of-presubmit', result=False, presubmit=True) |
| RESULT_FAILURE = _cbuild_results( |
| 'hash-of-failure', result=False, regression=False) |
| RESULT_BRANCH = _cbuild_results('hash-of-branch', branch='branch-name') |
| |
| CBUILD_RESULT_URLS = '''gs://dart-cbuild-test-results/hash-of-success |
| gs://dart-cbuild-test-results/ignore-this-directory/ |
| gs://dart-cbuild-test-results/hash-of-regression |
| gs://dart-cbuild-test-results/hash-of-presubmit |
| gs://dart-cbuild-test-results/hash-of-failure |
| gs://dart-cbuild-test-results/hash-of-branch |
| ''' |
| yield api.test( |
| 'with-results', |
| api.step_data( |
| 'gsutil find cbuild results', |
| stdout=api.raw_io.output_text(CBUILD_RESULT_URLS)), |
| RESULT_SUCCESS, |
| RESULT_REGRESSION, |
| RESULT_PRESUBMIT, |
| RESULT_FAILURE, |
| RESULT_BRANCH, |
| schedules_build('hash-of-success', True), |
| schedules_build('hash-of-regression', True), |
| schedules_build('hash-of-presubmit', False), |
| schedules_build('hash-of-failure', True), |
| schedules_build('hash-of-branch', False), |
| api.post_process(post_process.MustRun, |
| 'gsutil delete result for hash-of-success'), |
| api.post_process(post_process.MustRun, |
| 'gsutil delete result for hash-of-regression'), |
| api.post_process(post_process.MustRun, |
| 'gsutil delete result for hash-of-presubmit'), |
| api.post_process(post_process.MustRun, |
| 'gsutil delete result for hash-of-failure'), |
| api.post_process(post_process.MustRun, |
| 'gsutil delete result for hash-of-branch'), |
| ) |
| |
| yield api.test( |
| 'experimental-with-results', |
| api.runtime(is_experimental=True), |
| api.step_data( |
| 'gsutil find cbuild results', |
| stdout=api.raw_io.output_text(CBUILD_RESULT_URLS)), |
| RESULT_SUCCESS, |
| RESULT_REGRESSION, |
| RESULT_PRESUBMIT, |
| RESULT_FAILURE, |
| RESULT_BRANCH, |
| api.post_process(post_process.DoesNotRunRE, '.*(delete|update).*'), |
| ) |
