| # Copyright 2023 The Dart Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from recipe_engine.post_process import (MustRun, DropExpectation) |
| |
| from PB.recipes.dart.release.release import Release |
| |
| import datetime |
| import json |
| import re |
| |
| DEPS = [ |
| 'dart', |
| 'depot_tools/git', |
| 'depot_tools/gsutil', |
| 'recipe_engine/bcid_reporter', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/cipd', |
| 'recipe_engine/file', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/runtime', |
| 'recipe_engine/service_account', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| ] |
| |
| RELEASES = { |
| 'stable': 'stable', |
| 'beta': 'testing', |
| 'dev': 'unstable', |
| } |
| |
| PROPERTIES = Release |
| |
| PYTHON_VERSION_COMPATIBILITY = 'PY3' |
| |
| |
| def _install_reprepro(api): |
| api.step('install reprepro dependencies', [ |
| 'sudo', 'apt-get', 'install', '-y', 'libarchive-dev', 'libassuan-dev', |
| 'libdb-dev', 'libdb5.3-dev', 'libgpg-error-dev', 'libgpgme-dev' |
| ]) |
| reprepro_dir = api.path['cleanup'].join('reprepro') |
| # We pin a custom reprepro with special support for multiple versions. The git |
| # hash is the exact version we have been relying on locally and protects us |
| # against the third party repository. |
| api.git.checkout( |
| url='https://github.com/ionos-enterprise/reprepro', |
| ref='b37d8daba6bfb4c20241cf623a24e64532dd8868', |
| dir_path=reprepro_dir, |
| submodules=False, |
| set_got_revision=False, |
| submodule_update_recursive=False) |
| with api.context(cwd=reprepro_dir): |
| api.step('configure reprepro', ['./configure']) |
| api.step('make reprepro', ['make', '-j8']) |
| reprepro = reprepro_dir.join('reprepro') |
| return reprepro |
| |
| |
| def _get_latest_release(api, channel): |
| version_url = f'gs://dart-archive/channels/{channel}/release/latest/VERSION' |
| stdout = api.gsutil.cat( |
| version_url, |
| name=f'get latest {channel} version', |
| stdout=api.raw_io.output_text()).stdout |
| version = json.loads(stdout)['version'] |
| return api.dart.Version(version=version) |
| |
| |
| def _available_versions(api, channel): |
| horizon = api.dart.Version(version='2.0.0') |
| if channel != 'stable': |
| stable = _get_latest_release(api, 'stable') |
| latest = _get_latest_release(api, channel) |
| horizon = stable if stable < latest else latest |
| |
| releases_url = f'gs://dart-archive/channels/{channel}/release/' |
| listing = api.gsutil.list( |
| releases_url, |
| name=f'list {channel} releases', |
| stdout=api.raw_io.output_text(add_output_log=True)) |
| versions = [] |
| for url in listing.stdout.splitlines(): |
| name = url.split('/')[6] |
| if api.dart.is_version(name): |
| version = api.dart.Version(version=name) |
| if horizon <= version: |
| versions.append(version) |
| |
| return versions |
| |
| |
| def _download(api, channel, debs_dir): |
| provenance_horizon = api.dart.Version(version={ |
| 'dev': '3.1.0-145.0.dev', |
| 'beta': '3.1.0-163.1.beta', |
| 'stable': '3.0.3', |
| }[channel]) |
| api.file.ensure_directory('mkdir debs', debs_dir) |
| for version in _available_versions(api, channel): |
| # TODO: Detect all the available architectures. |
| deb_name = f'dart_{version}-1_amd64.deb' |
| deb_file = debs_dir.join(deb_name) |
| api.dart.download_and_verify( |
| deb_name, |
| 'dart-archive', |
| f'channels/{channel}/release/{version}/linux_packages/{deb_name}', |
| deb_file, |
| 'misc_software://dart/sdk/debian', |
| no_verify=version < provenance_horizon) |
| |
| |
| def _repository(api, reprepro, version, release, base_dir, debs_dir, |
| repository_dir): |
| conf_dir = base_dir.join('conf') |
| api.file.ensure_directory('mkdir conf', conf_dir) |
| distributions_file = conf_dir.join('distributions') |
| distributions = f'''Origin: Google LLC |
| Label: Google |
| Suite: {release} |
| Codename: {release} |
| Version: 1.0 |
| Components: main |
| Description: Google dart-linux software repository |
| Architectures: amd64 |
| |
| ''' |
| api.file.write_text('write conf/distributions', distributions_file, |
| distributions) |
| debs = api.file.listdir( |
| 'list debian packages', |
| debs_dir, |
| test_data=[ |
| f'dart_{version}-1_amd64.deb', |
| f'dart_{version}-1_amd64.deb.intoto.jsonl' |
| ]) |
| debs = [deb for deb in debs if str(deb).endswith('.deb')] |
| api.step('reprepro', [ |
| reprepro, '--outdir', repository_dir, '--basedir', base_dir, 'includedeb', |
| release |
| ] + debs) |
| |
| |
| def _sign(api, version, release_file): |
| # Delete the previous signing request to prevent signing the old version. |
| api.gsutil(['rm', '-r', 'gs://dart-debian-signing/**'], |
| name='delete contents of dart-debian-signing bucket', |
| ok_ret='any') |
| listing = api.gsutil.list( |
| 'gs://dart-debian-signing/', |
| name='list dart-debian-signing contents', |
| stdout=api.raw_io.output_text(add_output_log=True)).stdout |
| api.step.empty( |
| 'confirm the dart-debian-signing bucket is non-empty', |
| status=api.step.INFRA_FAILURE if listing else api.step.SUCCESS) |
| |
| # Prepare the signing request in cloud storage along with provenance proving |
| # the Release file was uploaded by this builder. |
| api.gsutil.upload( |
| release_file, 'dart-debian-signing', 'Release', name='upload Release') |
| sha256 = api.file.file_hash(release_file) |
| api.bcid_reporter.report_gcs(sha256, 'gs://dart-debian-signing/Release') |
| |
| # Trigger debian signing by committing to a repository polled by the service. |
| debian_signing_dir = api.path['cleanup'].join('debian-signing') |
| api.git.checkout( |
| url='https://dart-internal.googlesource.com/debian-signing', |
| ref='main', |
| dir_path=debian_signing_dir) |
| with api.context(cwd=debian_signing_dir): |
| api.git('config', 'user.name', 'Dart CI', name='configure user.name') |
| api.git( |
| 'config', |
| 'user.email', |
| api.buildbucket.swarming_task_service_account, |
| name='configure user.email') |
| api.git('commit', '--allow-empty', '-m', |
| f'Trigger {version} debian signing') |
| api.git( |
| 'push', 'origin', 'HEAD:refs/heads/main', name='trigger debian signing') |
| |
| # Wait for the signed Release to appear in cloud storage. |
| attempts = 0 |
| subdirectory = None |
| signature_object = None |
| provenance_object = None |
| while True: |
| listing = api.gsutil.list( |
| 'gs://dart-debian-signing/**', |
| name='await debian signing', |
| stdout=api.raw_io.output_text(add_output_log=True)) |
| |
| for url in listing.stdout.splitlines(): |
| object_path = url[len('gs://dart-debian-signing/'):] |
| if object_path.startswith('prod/dart/signing/debian/continuous/'): |
| if object_path.endswith('/Release.asc'): |
| signature_object = object_path |
| if object_path.endswith('.intoto.jsonl'): |
| provenance_object = object_path |
| |
| if signature_object and provenance_object: |
| break |
| |
| attempts += 1 |
| if 15 <= attempts: |
| api.step.empty( |
| 'timed out waiting for debian signing', status=api.step.INFRA_FAILURE) |
| api.time.sleep(60) |
| |
| # Download the signed Release and verify its provenance. |
| api.dart.download_and_verify( |
| 'Release.asc', |
| 'dart-debian-signing', |
| signature_object, |
| f'{release_file}.gpg', |
| 'misc_software://kokoro/test/generic_builder/allow_l2', |
| provenance_object=provenance_object) |
| # TODO: Clean up the bucket when this recipe goes into production. |
| #api.gsutil(['rm', '-r', 'gs://dart-debian-signing/**'], |
| # name='clean up dart-debian-signing bucket') |
| |
| |
| def _upload(api, repository_dir): |
| # TODO(b/231124800): Replace download.dartlang.org with a secure bucket. |
| api.gsutil( |
| [ |
| '-h', 'Cache-Control:private, max-age=0, no-cache', '-m', 'rsync', |
| '-R', repository_dir, 'gs://download.dartlang.org/linux/debian/' |
| ], |
| name='publish debian repository', |
| dry_run=api.runtime.is_experimental, |
| ) |
| |
| |
| def RunSteps(api, properties): |
| assert not api.buildbucket.builder_name.endswith( |
| '-try'), 'tryjob is not supported' |
| api.bcid_reporter.report_stage('start') |
| version = properties.version |
| assert version, 'the recipe requires a version' |
| channel = api.dart.Version(version=version).fields['CHANNEL'] |
| release = RELEASES[channel] |
| base_dir = api.path['cleanup'] |
| debs_dir = base_dir.join('debs') |
| repository_dir = base_dir.join('repository') |
| api.bcid_reporter.report_stage('fetch') |
| reprepro = _install_reprepro(api) |
| _download(api, channel, debs_dir) |
| api.bcid_reporter.report_stage('compile') |
| _repository(api, reprepro, version, release, base_dir, debs_dir, |
| repository_dir) |
| api.bcid_reporter.report_stage('upload') |
| release_file = repository_dir.join('dists').join(release).join('Release') |
| api.file.listdir('list repository', repository_dir, recursive=True) |
| api.file.read_text('read Release', release_file) |
| api.file.read_text( |
| 'read Packages', |
| repository_dir.join('dists').join(release).join('main').join( |
| 'binary-amd64').join('Packages')) |
| _sign(api, version, release_file) |
| _upload(api, repository_dir) |
| api.bcid_reporter.report_stage('upload-complete') |
| |
| |
| def GenTests(api): |
| build = api.buildbucket.ci_build_message( |
| project='dart-internal', |
| builder='sign-debian', |
| git_repo='https://dart.googlesource.com/sdk', |
| git_ref='refs/heads/dev', |
| revision='aa0ef37c3022a866a939f2040f8b9cf7b7b18d29') |
| api.buildbucket.update_backend_service_account(build, 'service@example.com') |
| |
| versions = ''' |
| gs://dart-archive/channels/dev/release/latest/ |
| gs://dart-archive/channels/dev/release/3.0.0-55.0.dev/ |
| gs://dart-archive/channels/dev/release/3.1.0-145.0.dev/ |
| gs://dart-archive/channels/dev/release/3.1.0-155.0.dev/ |
| gs://dart-archive/channels/dev/release/12345/ |
| '''[1:] |
| yield api.test( |
| 'sign', |
| api.properties(Release(version='3.1.0-155.0.dev')), |
| api.buildbucket.build(build), |
| api.step_data( |
| 'gsutil get latest stable version', |
| stdout=api.raw_io.output_text('{"version": "3.0.5"}')), |
| api.step_data( |
| 'gsutil get latest dev version', |
| stdout=api.raw_io.output_text('{"version": "3.1.0-155.0.dev"}')), |
| api.step_data( |
| 'gsutil list dev releases', stdout=api.raw_io.output_text(versions)), |
| api.step_data( |
| 'verify dart_3.1.0-145.0.dev-1_amd64.deb provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.step_data( |
| 'verify dart_3.1.0-155.0.dev-1_amd64.deb provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.step_data( |
| 'gsutil await debian signing (2)', |
| stdout=api.raw_io.output_text(''' |
| gs://dart-debian-signing/Release |
| gs://dart-debian-signing/Release.attestation |
| gs://dart-debian-signing/Release.intoto.jsonl |
| gs://dart-debian-signing/prod/dart/signing/debian/continuous/3/20230616-035526/artifacts/Release.asc |
| gs://dart-debian-signing/prod/dart/signing/debian/continuous/3/20230616-035526/d0395538-7394-4cce-a1d4-7182ad02e19b.intoto.jsonl |
| '''[1:])), |
| api.step_data( |
| 'verify Release.asc provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.post_process(MustRun, 'snoop: report_stage (5)'), |
| ) |
| yield api.test( |
| 'sign-timeout', |
| api.properties(Release(version='3.1.0-155.0.dev')), |
| api.buildbucket.build(build), |
| api.step_data( |
| 'gsutil get latest stable version', |
| stdout=api.raw_io.output_text('{"version": "3.0.5"}')), |
| api.step_data( |
| 'gsutil get latest dev version', |
| stdout=api.raw_io.output_text('{"version": "3.1.0-155.0.dev"}')), |
| api.step_data( |
| 'gsutil list dev releases', stdout=api.raw_io.output_text(versions)), |
| api.step_data( |
| 'verify dart_3.1.0-145.0.dev-1_amd64.deb provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.step_data( |
| 'verify dart_3.1.0-155.0.dev-1_amd64.deb provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.post_process(DropExpectation), |
| status='INFRA_FAILURE', |
| ) |
