| # Copyright 2022 The Dart Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import datetime, json, re |
| |
| from recipe_engine import post_process |
| |
| from PB.go.chromium.org.luci.buildbucket.proto import common as common_pb2 |
| |
| from PB.recipe_engine import result as result_pb2 |
| |
| from PB.recipes.dart.release.release import Release |
| |
| DEPS = [ |
| 'dart', |
| 'depot_tools/git', |
| 'depot_tools/gsutil', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/runtime', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| ] |
| PROPERTIES = Release |
| REPO_URL = 'https://dart.googlesource.com/sdk' |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| |
| def _get_latest_release(api, channel): |
| version_url = f'gs://dart-archive/channels/{channel}/release/latest/VERSION' |
| stdout = api.gsutil.cat( |
| version_url, |
| name=f'get latest {channel} version', |
| stdout=api.raw_io.output_text()).stdout |
| version = json.loads(stdout)['version'] |
| return api.dart.Version(version=version) |
| |
| |
| def _is_already_signed(api, version, channel, zip_name): |
| return api.gsutil.stat( |
| f'gs://dart-archive/channels/{channel}/signed/{version}/sdk/{zip_name}', |
| name=f'check if {zip_name} {version} is already signed', |
| ok_ret=[0, 1]).exc_result.retcode == 0 |
| |
| |
| # Tag the release if it hasn't already been tagged. |
| def _tag(api, properties, version, channel): |
| api.git.checkout( |
| REPO_URL, |
| submodules=False, |
| submodule_update_recursive=False, |
| ) |
| with api.context(cwd=api.path['checkout']): |
| api.git('config', 'user.name', 'Dart CI', name='configure user.name') |
| api.git( |
| 'config', |
| 'user.email', |
| api.buildbucket.swarming_task_service_account, |
| name='configure user.email') |
| api.git('fetch', 'origin') |
| result = api.git( |
| 'rev-parse', |
| '--verify', |
| '-q', |
| f'{version}^{{}}', |
| name=f'find existing tag', |
| stdout=api.raw_io.output_text(add_output_log=True), |
| ok_ret=[0, 1], # rev-parse returns 1 if no such tag. |
| ) |
| if result.exc_result.retcode == 0: |
| # The release is already tagged. |
| return result.stdout.rstrip() |
| version_escaped = version.replace('.', '\\.') |
| commit = api.git( |
| 'rev-list', |
| '-n', |
| '1', |
| f'--grep=Version {version_escaped}$', |
| f'origin/{channel}', |
| name=f'find commit', |
| stdout=api.raw_io.output_text(add_output_log=True)).stdout.rstrip() |
| api.git('tag', '--annotate', f'--message={version}', version, commit) |
| push_args = ['push'] |
| if api.runtime.is_experimental: |
| push_args.append('--dry-run') |
| push_args += ['origin', f'refs/tags/{version}'] |
| api.git(*push_args, name=f'push tag') |
| return commit |
| |
| |
| def _trigger(api, builder, gitiles_commit, properties=None): |
| return api.buildbucket.schedule( |
| [ |
| api.buildbucket.schedule_request( |
| project='dart-internal', |
| bucket='ci', |
| builder=builder, |
| properties=properties, |
| gitiles_commit=gitiles_commit) |
| ], |
| step_name=f'trigger {builder}', |
| )[0] |
| |
| |
| def _collect(api, build): |
| return api.buildbucket.collect_build( |
| build.id, |
| raise_if_unsuccessful=True, |
| step_name=f'await {build.builder.builder}') |
| |
| |
| def RunSteps(api, properties): |
| assert not api.buildbucket.builder_name.endswith( |
| '-try'), 'tryjob is not supported' |
| version = properties.version |
| assert version, 'the recipe requires a version' |
| |
| if version == 'dev': |
| version_string = api.gsutil.cat( |
| 'gs://dart-internal-roll/version', |
| name=f'get latest roll', |
| stdout=api.raw_io.output_text()).stdout.rstrip() |
| if not api.dart.is_version(version_string): |
| return result_pb2.RawResult( |
| summary_markdown=f'{version_string} must be a valid version', |
| status=common_pb2.FAILURE) |
| available = api.dart.Version(version_string) |
| if available.channel != 'dev': |
| return result_pb2.RawResult( |
| summary_markdown=f'{version_string} must be on the dev channel', |
| status=common_pb2.FAILURE) |
| latest = _get_latest_release(api, 'dev') |
| if available <= latest: |
| summary = f'{available} has already been published' |
| api.step.empty(summary) |
| return result_pb2.RawResult( |
| summary_markdown=summary, status=common_pb2.SUCCESS) |
| version = str(available) |
| |
| api.step.empty(f'publishing {version}') |
| api.step.active_result.presentation.properties['version'] = version |
| channel = api.dart.Version(version=version).fields['CHANNEL'] |
| |
| is_retrying = set() |
| |
| # Tag the release commit if it hasn't already been tagged. |
| commit = _tag(api, properties, version, channel) |
| |
| gitiles_commit = common_pb2.GitilesCommit( |
| host='dart.googlesource.com', |
| project='sdk', |
| ref=f'refs/heads/{channel}', |
| id=commit, |
| ) |
| |
| @api.time.exponential_retry(2, datetime.timedelta(minutes=2)) |
| def _retry_collect(api, build): |
| if build.builder.builder in is_retrying: |
| build = _trigger(api, build.builder.builder, gitiles_commit, |
| build.input.properties) |
| is_retrying.add(build.builder.builder) |
| _collect(api, build) |
| |
| def _retry_trigger(api, builder, gitiles_commit, properties=None): |
| build = _trigger(api, builder, gitiles_commit, properties) |
| _retry_collect(api, build) |
| |
| # Sign macOS SDKs if not already done. |
| if not _is_already_signed(api, f'hash/{commit}', channel, |
| 'dartsdk-macos-x64-release.zip') or \ |
| not _is_already_signed(api, f'hash/{commit}', channel, |
| 'dartsdk-macos-arm64-release.zip'): |
| _retry_trigger(api, 'sign-mac', gitiles_commit) |
| |
| # Promote the builds to the official release channel. |
| _retry_trigger( |
| api, 'promote', gitiles_commit, properties={'version': version}) |
| |
| # Publish to docker. |
| docker = _trigger( |
| api, 'docker', gitiles_commit, properties={'version': version}) |
| |
| # Publish to homebrew. |
| homebrew = _trigger(api, 'homebrew', gitiles_commit) |
| |
| # Publish dartdocs. |
| dartdoc = _trigger( |
| api, 'dartdoc', gitiles_commit, properties={'version': version}) |
| |
| # Publish to chocolatey. |
| chocolatey = _trigger(api, 'chocolatey', gitiles_commit) |
| |
| # Sign debian repository. |
| sign_debian = _trigger( |
| api, 'sign-debian', gitiles_commit, properties={'version': version}) |
| |
| # Collect parallel builds. |
| _retry_collect(api, docker) |
| _retry_collect(api, homebrew) |
| _retry_collect(api, dartdoc) |
| _retry_collect(api, chocolatey) |
| _retry_collect(api, sign_debian) |
| |
| return result_pb2.RawResult( |
| summary_markdown=f'published {version}', status=common_pb2.SUCCESS) |
| |
| |
| def GenTests(api): |
| buildbucket_build = api.buildbucket.ci_build_message() |
| api.buildbucket.update_backend_service_account(buildbucket_build, |
| 'service@example.com') |
| yield api.test( |
| 'stable', |
| api.properties(Release(version='2.18.5')), |
| api.buildbucket.build(buildbucket_build), |
| api.step_data('find existing tag', retcode=1), |
| api.step_data( |
| 'find commit', |
| stdout=api.raw_io.output_text( |
| '1f8ac10f23c5b5bc1167bda84b833e5c057a77d2\n')), |
| api.step_data( |
| 'gsutil check if dartsdk-macos-arm64-release.zip ' |
| 'hash/1f8ac10f23c5b5bc1167bda84b833e5c057a77d2 is already signed', |
| retcode=1), |
| api.buildbucket.simulated_collect_output( |
| [ |
| api.buildbucket.ci_build_message( |
| build_id=8922054662172514000, status='INFRA_FAILURE'), |
| ], |
| step_name='await sign-mac', |
| ), |
| api.post_process(post_process.MustRun, 'push tag'), |
| api.post_process(post_process.MustRun, 'trigger promote'), |
| api.post_process(post_process.MustRun, 'trigger chocolatey'), |
| ) |
| yield api.test( |
| 'retry', |
| api.properties(Release(version='2.18.5')), |
| api.buildbucket.build(buildbucket_build), |
| api.step_data( |
| 'find existing tag', |
| stdout=api.raw_io.output_text( |
| '1f8ac10f23c5b5bc1167bda84b833e5c057a77d2\n')), |
| api.post_process(post_process.MustRun, 'trigger promote'), |
| api.post_process(post_process.MustRun, 'trigger chocolatey'), |
| api.post_process(post_process.DropExpectation), |
| ) |
| yield api.test( |
| 'dev', |
| api.properties(Release(version='dev')), |
| api.buildbucket.build(buildbucket_build), |
| api.step_data( |
| 'find existing tag', |
| stdout=api.raw_io.output_text( |
| '1f8ac10f23c5b5bc1167bda84b833e5c057a77d2\n')), |
| api.step_data( |
| 'gsutil get latest roll', |
| stdout=api.raw_io.output_text('2.18.0-271.0.dev\n')), |
| api.step_data( |
| 'gsutil get latest dev version', |
| stdout=api.raw_io.output_text('{"version": "2.18.0-270.0.dev"}')), |
| api.post_process(post_process.DoesNotRunRE, '(push|tag).*'), |
| api.post_process(post_process.MustRun, 'trigger promote'), |
| api.post_process(post_process.MustRun, 'trigger chocolatey'), |
| ) |
| yield api.test( |
| 'dev-already', |
| api.properties(Release(version='dev')), |
| api.buildbucket.build(buildbucket_build), |
| api.step_data( |
| 'gsutil get latest roll', |
| stdout=api.raw_io.output_text('2.18.0-271.0.dev\n')), |
| api.step_data( |
| 'gsutil get latest dev version', |
| stdout=api.raw_io.output_text('{"version": "2.18.0-271.0.dev"}')), |
| api.post_process(post_process.DoesNotRunRE, 'trigger.*'), |
| api.post_process(post_process.DropExpectation), |
| ) |
| yield api.test( |
| 'dev-is-main', |
| api.properties(Release(version='dev')), |
| api.buildbucket.build(buildbucket_build), |
| api.step_data( |
| 'gsutil get latest roll', |
| stdout=api.raw_io.output_text('Main hash: abcdef01\n')), |
| api.post_process(post_process.DropExpectation), |
| status='FAILURE', |
| ) |
| yield api.test( |
| 'dev-is-stable', |
| api.properties(Release(version='dev')), |
| api.buildbucket.build(buildbucket_build), |
| api.step_data( |
| 'gsutil get latest roll', stdout=api.raw_io.output_text('2.18.5\n')), |
| api.post_process(post_process.DropExpectation), |
| status='FAILURE', |
| ) |
| yield api.test( |
| 'dry-run', |
| api.runtime(is_experimental=True), |
| api.properties(Release(version='2.18.5')), |
| api.buildbucket.build(buildbucket_build), |
| api.step_data('find existing tag', retcode=1), |
| api.step_data( |
| 'find commit', |
| stdout=api.raw_io.output_text( |
| '1f8ac10f23c5b5bc1167bda84b833e5c057a77d2\n')), |
| api.post_process(post_process.MustRun, 'push tag'), |
| api.post_process(post_process.DropExpectation), |
| ) |
