| # Copyright 2022 The Dart Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import json |
| import re |
| |
| from PB.recipes.dart.release.release import Release |
| |
| STORAGE = 'gs://dart-archive/channels' |
| |
| DEPS = [ |
| 'dart', |
| 'depot_tools/gsutil', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/cipd', |
| 'recipe_engine/file', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/runtime', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| ] |
| |
| PYTHON_VERSION_COMPATIBILITY = 'PY3' |
| |
| PROPERTIES = Release |
| |
| |
| def _get_latest_release(api, channel): |
| version_url = f'{STORAGE}/{channel}/release/latest/VERSION' |
| stdout = api.gsutil.cat( |
| version_url, |
| name=f'get latest {channel} version', |
| stdout=api.raw_io.output_text()).stdout |
| version = json.loads(stdout)['version'] |
| return api.dart.Version(version=version) |
| |
| |
| def _safety_check_on_gs_path(api, gs_path, version, channel): |
| if not (version != None and len(channel) > 0 and |
| ('%s' % version) in gs_path and |
| channel in gs_path): # pragma: no cover |
| raise api.step.StepFailure(f'Sanity check failed on GS URI: {gs_path}') |
| |
| |
| def _exists(api, gs_path, name): |
| return api.gsutil(['ls', gs_path], name=name, |
| ok_ret='any').exc_result.retcode == 0 |
| |
| |
| # Google cloud storage has read-after-write, read-after-update, and |
| # read-after-delete consistency, but not list after delete consistency. Because |
| # gsutil uses list to figure out if it should do the unix styly copy to or copy |
| # into, this means that if the directory is reported as still being there (after |
| # it has been deleted) gsutil will copy into the directory instead of to the |
| # directory. |
| def _wait_for_delete_to_be_consistent_with_list(api, gs_path, name): |
| if api.runtime.is_experimental: |
| return |
| while _exists(api, gs_path, f'check if {name} still exists'): |
| api.time.sleep(1) |
| |
| |
| def _gsutil(api, args, name): |
| api.gsutil(args, name=name, dry_run=api.runtime.is_experimental) |
| |
| |
| def _remove_gs_directory(api, gs_path, version, channel, name): |
| _safety_check_on_gs_path(api, gs_path, version, channel) |
| if _exists(api, gs_path, f'check if {name} already exists'): |
| _gsutil(api, ['-m', 'rm', '-R', '-f', gs_path], f'delete old {name}') |
| _wait_for_delete_to_be_consistent_with_list(api, gs_path, name) |
| |
| |
| def _download(api, channel, commit, version, directory): |
| # Download the signed sdk directory. |
| from_loc = f'{STORAGE}/{channel}/signed/hash/{commit}/sdk' |
| if _exists(api, from_loc, 'check if sdk is signed'): |
| api.gsutil(['-m', 'cp', '-R', from_loc, directory], |
| name='download signed sdk') |
| |
| # Download the unsigned sdk directory without clobbering signed files. |
| from_loc = f'{STORAGE}/{channel}/raw/hash/{commit}/sdk' |
| api.gsutil(['-m', 'cp', '-n', '-R', from_loc, directory], name='download sdk') |
| |
| # Download the other artifacts. |
| for artifact in ['api-docs', 'linux_packages', 'VERSION']: |
| from_loc = f'{STORAGE}/{channel}/raw/hash/{commit}/{artifact}' |
| api.gsutil(['-m', 'cp', '-R', from_loc, directory], |
| name=f'download {artifact}') |
| |
| # Double check the build is the intended version. |
| downloaded_version = json.loads( |
| api.file.read_text('check VERSION', directory.join('VERSION')))['version'] |
| assert str(version) == downloaded_version, 'the build is the intended version' |
| |
| |
| def _verify(api, version, directory): |
| # Simulate the file listing data accurately during recipe testing. |
| test_files = [ |
| 'VERSION', 'api-docs/dartdocs-gen-api.zip', |
| 'api-docs/dartdocs-gen-api.zip.attestation', |
| 'api-docs/dartdocs-gen-api.zip.intoto.jsonl' |
| ] |
| test_files.extend([ |
| f'linux_packages/dart_{version}-1.debian.tar.xz', |
| f'linux_packages/dart_{version}-1.dsc', |
| f'linux_packages/dart_{version}-1_amd64.deb', |
| f'linux_packages/dart_{version}.orig.tar.gz', |
| ]) |
| for platform in ['linux-riscv64', 'macos-arm64', 'windows-x64']: |
| for ext in ['', '.attestation', '.intoto.jsonl', '.sha256sum']: |
| test_files.append(f'sdk/dartsdk-{platform}-release.zip{ext}') |
| |
| files = api.file.listdir( |
| 'list release files', directory, recursive=True, test_data=test_files) |
| for file in files: |
| name = str(file)[len(str(directory)) + 1:] |
| if (name == 'VERSION' or name.endswith('.sha256sum') or |
| name.endswith('.attestation') or name.endswith('.intoto.jsonl')): |
| continue |
| resources = { |
| 'linux': 'sdk/linux', |
| 'linux_packages': 'sdk/debian', |
| 'macos': 'sdk/macos', |
| 'windows': 'sdk/windows', |
| 'api-docs': 'apidocs', |
| } |
| resource = None |
| for substring, suffix in resources.items(): |
| if substring in name: |
| resource = f'misc_software://dart/{suffix}' |
| assert resource |
| verification = api.dart.verify_provenance(name, file, resource) |
| vsa = verification['verificationSummary'] |
| api.file.write_text(f'write {name}.vsa.intoto.jsonl', |
| directory.join(f'{name}.vsa.intoto.jsonl'), vsa) |
| |
| |
| def _upload(api, channel, version, directory): |
| # Upload the artifacts. |
| for artifact in ['sdk', 'api-docs', 'linux_packages', 'VERSION']: |
| from_loc = directory.join(artifact) |
| object_path = f'{channel}/release/{version}/{artifact}' |
| to_loc = f'{STORAGE}/{object_path}' |
| # Remove any artifacts that are already there. This will primarily happen |
| # for the latest directory which contains the previous release. However, |
| # ensure VERSION exists at all times (without caching in latest) as the |
| # website uses it to show which release is the latest. |
| no_cache = [] |
| if artifact != 'VERSION': |
| _remove_gs_directory(api, to_loc, version, channel, object_path) |
| elif version == 'latest': |
| no_cache = ['-h', 'Cache-Control: no-cache'] |
| _gsutil(api, no_cache + ['-m', 'cp', '-R', from_loc, to_loc], |
| f'upload {object_path}') |
| |
| |
| def _promote(api, channel, version, directory, to_channel): |
| _upload(api, to_channel, str(version), directory) |
| # Promote to latest unless it's an older version. |
| if _get_latest_release(api, to_channel) <= version: |
| _upload(api, to_channel, 'latest', directory) |
| # Promote beta to stable if stable becomes ahead of beta. |
| if to_channel == 'stable' and _get_latest_release(api, 'beta') <= version: |
| _promote(api, 'stable', version, directory, 'beta') |
| |
| |
| def _tag_cipd(api, version, directory): |
| git_revision = json.loads( |
| api.file.read_text('read VERSION', directory.join('VERSION')))['revision'] |
| test_files = [ |
| 'sdk/dartsdk-windows-x64-release.zip', |
| 'sdk/dartsdk-windows-x64-release.zip.sha256sum' |
| ] |
| files = api.file.listdir( |
| 'list release platforms', directory, recursive=True, test_data=test_files) |
| for file in files: |
| name = str(file)[len(str(directory)) + 1:] |
| match = re.match('^sdk/dartsdk-([^-]+)-([^-]+)-release.zip$', name) |
| if match: |
| cipd_os = api.dart.os_to_cipd_os(match[1]) |
| cipd_arch = api.dart.arch_to_cipd_arch(match[2]) |
| cipd_platform = f'{cipd_os}-{cipd_arch}' |
| cipd_name = f'dart/dart-sdk/{cipd_platform}' |
| # Set the version tag now that we commit to the package as that version. |
| api.cipd.set_tag(cipd_name, f'git_revision:{git_revision}', |
| {'version': str(version)}) |
| # Forward the cipd channel ref if the new version is newer. |
| try: |
| old_ref = api.cipd.describe( |
| cipd_name, version.channel, test_data_tags=['version:2.0.0']) |
| except api.cipd.Error: # pragma: no cover |
| old_ref = None |
| old_version = api.dart.Version( |
| version=next( |
| tag.tag.split(':')[1] |
| for tag in old_ref.tags |
| if tag.tag.startswith('version:'))) if old_ref else None |
| if not old_ref or old_version < version: |
| api.cipd.set_ref(cipd_name, f'git_revision:{git_revision}', |
| [version.channel]) |
| |
| |
| def RunSteps(api, properties): |
| assert not api.buildbucket.builder_name.endswith( |
| '-try'), 'tryjob is not supported' |
| version_string = properties.version |
| assert version_string, 'recipes require a version property' |
| version = api.dart.Version(version=version_string) |
| channel = version.fields['CHANNEL'] |
| commit = api.buildbucket.gitiles_commit.id |
| directory = api.path['cleanup'].join('release') |
| api.file.ensure_directory('create release directory', directory) |
| _download(api, channel, commit, version, directory) |
| _verify(api, version, directory) |
| _promote(api, channel, version, directory, channel) |
| _tag_cipd(api, version, directory) |
| |
| |
| def GenTests(api): |
| verified = '{"allowed": true,"verificationSummary":"{\\"payload\\": \\"\\"}"}' |
| yield api.test( |
| 'dev', |
| api.buildbucket.ci_build(), |
| api.properties(Release(version='2.17.0-99.0.dev')), |
| api.step_data( |
| 'gsutil check if dev/release/2.17.0-99.0.dev/sdk already exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if dev/release/2.17.0-99.0.dev/api-docs already exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if dev/release/2.17.0-99.0.dev/linux_packages already exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if dev/release/latest/sdk still exists (2)', retcode=1), |
| api.step_data( |
| 'gsutil check if dev/release/latest/api-docs still exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if dev/release/latest/linux_packages still exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil get latest dev version', |
| stdout=api.raw_io.output_text('{"version": "2.17.0-98.0.dev"}')), |
| api.step_data( |
| 'verify api-docs/dartdocs-gen-api.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-macos-arm64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-linux-riscv64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-windows-x64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.17.0-99.0.dev-1.debian.tar.xz provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.17.0-99.0.dev-1.dsc provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.17.0-99.0.dev-1_amd64.deb provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.17.0-99.0.dev.orig.tar.gz provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data('check VERSION', |
| api.file.read_text('{"version": "2.17.0-99.0.dev"}')), |
| api.step_data('read VERSION', |
| api.file.read_text('{"revision": "abcdef01"}')), |
| ) |
| yield api.test( |
| 'beta', |
| api.buildbucket.ci_build(), |
| api.properties(Release(version='2.17.0-69.1.beta')), |
| api.step_data( |
| 'gsutil check if beta/release/2.17.0-69.1.beta/sdk already exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if beta/release/2.17.0-69.1.beta/api-docs already exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if beta/release/2.17.0-69.1.beta/linux_packages already exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if beta/release/latest/sdk still exists', retcode=1), |
| api.step_data( |
| 'gsutil check if beta/release/latest/api-docs still exists', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if beta/release/latest/linux_packages still exists', |
| retcode=1), |
| api.step_data( |
| 'verify api-docs/dartdocs-gen-api.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-macos-arm64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-linux-riscv64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-windows-x64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.17.0-69.1.beta-1.debian.tar.xz provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.17.0-69.1.beta-1.dsc provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.17.0-69.1.beta-1_amd64.deb provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.17.0-69.1.beta.orig.tar.gz provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data('check VERSION', |
| api.file.read_text('{"version": "2.17.0-69.1.beta"}')), |
| api.step_data( |
| 'gsutil get latest beta version', |
| stdout=api.raw_io.output_text('{"version": "2.17.0-69.1.beta"}')), |
| api.step_data('read VERSION', |
| api.file.read_text('{"revision": "abcdef01"}')), |
| ) |
| yield api.test( |
| 'dry', |
| api.buildbucket.ci_build(), |
| api.runtime(is_experimental=True), |
| api.properties(Release(version='2.16.0')), |
| api.step_data( |
| 'verify api-docs/dartdocs-gen-api.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-macos-arm64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-linux-riscv64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify sdk/dartsdk-windows-x64-release.zip provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.16.0-1.debian.tar.xz provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.16.0-1.dsc provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.16.0-1_amd64.deb provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data( |
| 'verify linux_packages/dart_2.16.0.orig.tar.gz provenance', |
| stdout=api.raw_io.output_text(verified)), |
| api.step_data('check VERSION', |
| api.file.read_text('{"version": "2.16.0"}')), |
| api.step_data( |
| 'gsutil get latest stable version', |
| stdout=api.raw_io.output_text('{"version": "2.15.0"}')), |
| api.step_data( |
| 'gsutil get latest beta version', |
| stdout=api.raw_io.output_text('{"version": "2.15.0-12.34.beta"}')), |
| api.step_data( |
| 'gsutil get latest beta version (2)', |
| stdout=api.raw_io.output_text('{"version": "2.15.0-12.34.beta"}')), |
| api.step_data('read VERSION', |
| api.file.read_text('{"revision": "abcdef01"}')), |
| ) |
