| # Copyright 2023 The Dart Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| from recipe_engine.post_process import (MustRun, DoesNotRunRE, DropExpectation) |
| |
| from PB.recipes.dart.release.sign import Sign |
| |
| import datetime |
| import json |
| |
| DEPS = [ |
| 'dart', |
| 'depot_tools/osx_sdk', |
| 'depot_tools/gsutil', |
| 'recipe_engine/archive', |
| 'recipe_engine/bcid_reporter', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/cipd', |
| 'recipe_engine/file', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/runtime', |
| 'recipe_engine/service_account', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| ] |
| |
| ARCHS = ['x64', 'arm64'] |
| IDENTITY = 'Developer ID Application: FLUTTER.IO LLC (S8QB4VV633)' |
| BINARIES = [ |
| 'bin/dart', 'bin/dartaotruntime', 'bin/utils/gen_snapshot', |
| 'bin/utils/wasm-opt' |
| ] |
| PROPERTIES = Sign |
| |
| PYTHON_VERSION_COMPATIBILITY = 'PY3' |
| |
| |
| def _arch_to_cipd_arch(arch): |
| return { |
| 'ia32': '386', |
| 'x64': 'amd64', |
| 'arm': 'armv6l', |
| }.get(arch, arch) |
| |
| |
| def _is_already_signed(api, version, channel, zip_name): |
| return api.gsutil.stat( |
| f'gs://dart-archive/channels/{channel}/signed/{version}/sdk/{zip_name}', |
| name=f'check if {zip_name} {version} is already signed', |
| ok_ret=[0, 1]).exc_result.retcode == 0 |
| |
| |
| def _get_secret(api, name, object_path=None, extension='.key'): |
| return api.dart.get_secret( |
| name, |
| bucket='flutter_configs', |
| keyring='projects/flutter-infra-staging/locations/global' |
| '/keyRings/luci/cryptoKeys/flutter-infra', |
| object_path=object_path, |
| extension=extension) |
| |
| |
| def _get_commit_of_version(api, version, channel): |
| return json.loads( |
| api.gsutil.cat( |
| f'gs://dart-archive/channels/{channel}/raw/{version}/VERSION', |
| name=f'read {version} VERSION', |
| stdout=api.raw_io.output_text()).stdout)['revision'] |
| |
| |
| def _download(api, version, channel, directory, zip_name): |
| zip_file = directory.join(zip_name) |
| api.dart.download_and_verify( |
| f'raw {zip_name} {version}', 'dart-archive', |
| f'channels/{channel}/raw/{version}/sdk/{zip_name}', zip_file, |
| 'misc_software://dart/sdk/macos') |
| |
| |
| def _codesign(api, raw_dir, signed_dir, notary_dir, arch): |
| zip_basename = f'dartsdk-macos-{arch}-release' |
| zip_name = f'{zip_basename}.zip' |
| zip_file = raw_dir.join(zip_name) |
| notary_subdir = notary_dir.join(zip_basename) |
| api.file.ensure_directory(f'mkdir notary/{zip_basename}', notary_subdir) |
| api.archive.extract(f'extract dart-sdk from raw {zip_name}', zip_file, |
| api.path['cleanup']) |
| for binary in BINARIES: |
| program = api.path['cleanup'].join(f'dart-sdk/{binary}') |
| api.path.mock_add_paths(program) |
| if api.path.exists(program): |
| basename = api.path.basename(program) |
| entitlements = api.resource(f'Entitlements_{basename}.plist') |
| command = [ |
| '/usr/bin/codesign', '-f', '-s', IDENTITY, '--timestamp', |
| '--options=runtime', '--verbose' |
| ] |
| api.path.mock_add_paths(entitlements) |
| if api.path.exists(entitlements): |
| command.append(f'--entitlements={entitlements}') |
| command.append(program) |
| program_id = f'{zip_basename}/{basename}' |
| api.step(f'codesign {program_id}', command) |
| api.step(f'verify {program_id}', ['codesign', '-v', '--verbose', program]) |
| api.file.copy(f'save {program_id} for notarization', program, |
| notary_subdir.join(basename)) |
| dart_sdk_path = api.path['cleanup'].join('dart-sdk') |
| (api.archive.package(api.path['cleanup']).with_dir(dart_sdk_path).archive( |
| f'zip signed {zip_name}', signed_dir.join(zip_name))) |
| cipd_platform = 'mac-' + _arch_to_cipd_arch(arch) |
| cipd_file = signed_dir.join(f'{cipd_platform}.zip') |
| package_name = f'dart/dart-sdk/{cipd_platform}' |
| pkg = api.cipd.PackageDefinition( |
| package_name, dart_sdk_path, 'copy', preserve_writable=True) |
| pkg.add_dir(dart_sdk_path) |
| api.cipd.build_from_pkg(pkg, cipd_file) |
| api.file.rmtree(f'delete dart-sdk from raw {zip_name}', |
| api.path['cleanup'].join('dart-sdk')) |
| |
| |
| def _notarize(api, notary_dir, password_file, apple_id_file, team_id_file): |
| notary_zip = f'{notary_dir}.zip' |
| (api.archive.package(notary_dir).with_dir(notary_dir).archive( |
| f'zip for notarization', notary_zip)) |
| with api.osx_sdk('mac'): |
| api.step('notarize', [ |
| 'sh', |
| api.resource('notarize.sh'), password_file, apple_id_file, team_id_file, |
| 'xcrun', 'notarytool', 'submit', notary_zip, '--wait' |
| ]) |
| |
| |
| def _sign(api, archs, raw_dir, signed_dir, notary_dir, password_file, |
| apple_id_file, team_id_file, codesign_key): |
| try: |
| api.step( |
| 'delete previous keychain', |
| ['security', 'delete-keychain', 'build.keychain'], |
| ok_ret='any') |
| api.step('create keychain', |
| ['security', 'create-keychain', '-p', '', 'build.keychain']) |
| api.step('keychain settings', |
| ['security', 'set-keychain-settings', 'build.keychain']) |
| api.step('default keychain', |
| ['security', 'default-keychain', '-s', 'build.keychain']) |
| api.step('unlock keychain', |
| ['security', 'unlock-keychain', '-p', '', 'build.keychain']) |
| api.step('import certificate keychain', [ |
| 'security', 'import', codesign_key, '-k', 'build.keychain', '-P', |
| 'password', '-T', '/usr/bin/codesign' |
| ]) |
| api.step('set key partition list', [ |
| 'security', 'set-key-partition-list', '-S', |
| 'apple-tool:,apple:,codesign:', '-s', '-k', '', 'build.keychain' |
| ]) |
| api.step('list-keychains', ['security', 'list-keychains', '-d', 'user']) |
| identities = api.step( |
| 'find-identity', ['security', 'find-identity', '-v'], |
| stdout=api.raw_io.output_text(add_output_log=True)).stdout |
| if not IDENTITY in identities: |
| raise api.step.InfraFailure(f'Failed to import {IDENTITY}') |
| for arch in archs: |
| _codesign(api, raw_dir, signed_dir, notary_dir, arch) |
| _notarize(api, notary_dir, password_file, apple_id_file, team_id_file) |
| finally: |
| api.step( |
| 'lock keychain', ['security', 'lock-keychain', 'build.keychain'], |
| ok_ret='any') |
| api.step( |
| 'delete keychain', ['security', 'delete-keychain', 'build.keychain'], |
| ok_ret='any') |
| api.step('restore default keychain', |
| ['security', 'default-keychain', '-s', 'login.keychain']) |
| |
| |
| def _upload(api, version, commit, channel, directory, arch): |
| zip_name = f'dartsdk-macos-{arch}-release.zip' |
| zip_file = directory.join(zip_name) |
| sha256 = api.file.file_hash(zip_file) |
| sha256_name = zip_name + '.sha256sum' |
| sha256_path = directory.join(sha256_name) |
| api.file.write_text(f'write {sha256_name}', sha256_path, |
| f'{sha256} *{zip_name}\n') |
| bucket = 'dart-archive' |
| object_path = f'channels/{channel}/signed/{version}/sdk/{zip_name}' |
| api.gsutil.upload( |
| zip_file, bucket, object_path, name=f'upload signed {zip_name} {version}') |
| api.gsutil.upload( |
| sha256_path, |
| bucket, |
| object_path + '.sha256sum', |
| name=f'upload signed {sha256_name} {version}') |
| api.bcid_reporter.report_gcs(sha256, f'gs://{bucket}/{object_path}') |
| cipd_platform = 'mac-' + _arch_to_cipd_arch(arch) |
| cipd_file = directory.join(f'{cipd_platform}.zip') |
| package_name = f'dart/dart-sdk/{cipd_platform}' |
| pin = api.cipd.register( |
| package_name, cipd_file, tags={'git_revision': commit}) |
| sha256 = api.file.file_hash(cipd_file) |
| api.bcid_reporter.report_cipd(sha256, pin.package, pin.instance_id) |
| |
| |
| def RunSteps(api, properties): |
| assert not api.buildbucket.builder_name.endswith( |
| '-try'), 'tryjob is not supported' |
| api.bcid_reporter.report_stage('start') |
| version = properties.version |
| commit = None |
| if version: |
| channel = api.dart.Version(version=version).fields['CHANNEL'] |
| else: |
| commit = api.buildbucket.gitiles_commit.id |
| ref = api.buildbucket.gitiles_commit.ref |
| assert ref.startswith( |
| 'refs/heads/'), f'build must be triggered on a branch: {ref}' |
| channel = ref[len('refs/heads/'):] |
| version = f'hash/{commit}' |
| all_archs = properties.archs |
| assert all_archs, 'the recipe requires a list of archs' |
| api.bcid_reporter.report_stage('fetch') |
| with api.context(cwd=api.path['cleanup']): |
| all_signed = True |
| archs = [] |
| for arch in all_archs: |
| if api.runtime.is_experimental or \ |
| not _is_already_signed(api, version, channel, |
| f'dartsdk-macos-{arch}-release.zip'): |
| archs.append(arch) |
| if not archs: |
| api.step.empty(f'dart sdk {version} is already signed') |
| return |
| raw_dir = api.path['cleanup'].join('raw') |
| signed_dir = api.path['cleanup'].join('signed') |
| notary_dir = api.path['cleanup'].join('notary') |
| api.file.ensure_directory('mkdir raw', raw_dir) |
| api.file.ensure_directory('mkdir signed', signed_dir) |
| api.file.ensure_directory('mkdir notary', notary_dir) |
| if not commit: |
| commit = _get_commit_of_version(api, version, channel) |
| for arch in archs: |
| _download(api, version, channel, raw_dir, |
| f'dartsdk-macos-{arch}-release.zip') |
| password_file = _get_secret(api, 'codesign_app_specific_password') |
| apple_id_file = _get_secret(api, 'codesign_app_store_id') |
| team_id_file = _get_secret(api, 'codesign_team_id') |
| codesign_key = _get_secret( |
| api, 'codesign_key', object_path='dart_encrypted.p12', extension='.p12') |
| api.bcid_reporter.report_stage('compile') |
| _sign(api, archs, raw_dir, signed_dir, notary_dir, password_file, |
| apple_id_file, team_id_file, codesign_key) |
| api.bcid_reporter.report_stage('upload') |
| if not api.runtime.is_experimental: |
| for arch in archs: |
| _upload(api, version, commit, channel, signed_dir, arch) |
| api.bcid_reporter.report_stage('upload-complete') |
| |
| |
| def GenTests(api): |
| find_identity_bad = api.step_data( |
| 'find-identity', |
| stdout=api.raw_io.output_text(''' |
| 1) 72D8CF20531F3ABD251E5A15335E85F677A709E9 "MDM Client Identity Certificate" |
| 1 valid identities found |
| ''')) |
| find_identity = api.step_data( |
| 'find-identity', |
| stdout=api.raw_io.output_text(f''' |
| 1) E4CE0EFC4C211873BDBF547104E1F7FFEE0B70CF "{IDENTITY}" |
| 2) 72D8CF20531F3ABD251E5A15335E85F677A709E9 "MDM Client Identity Certificate" |
| 2 valid identities found |
| ''')) |
| ci_build = api.buildbucket.ci_build( |
| project='dart-internal', |
| builder='sign-mac', |
| git_repo='https://dart.googlesource.com/sdk', |
| git_ref='refs/heads/stable', |
| revision='abfdc3d50f6cf66165767da8df4f681a68467178') |
| version_data = api.step_data( |
| 'gsutil read 2.19.5 VERSION', |
| stdout=api.raw_io.output_text( |
| json.dumps({ |
| 'date': '2023-03-20', |
| 'version': '2.19.5', |
| 'revision': 'a318303f460e3c9d91e9aa2bdc73040ccbb885c6', |
| }))) |
| yield api.test( |
| 'sign', |
| api.properties(Sign(archs=ARCHS)), |
| ci_build, |
| api.step_data( |
| 'gsutil check if dartsdk-macos-x64-release.zip ' |
| 'hash/abfdc3d50f6cf66165767da8df4f681a68467178 is already signed', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if dartsdk-macos-arm64-release.zip ' |
| 'hash/abfdc3d50f6cf66165767da8df4f681a68467178 is already signed', |
| retcode=1), |
| api.step_data( |
| 'verify raw dartsdk-macos-x64-release.zip hash/abfdc3d50f6cf66165767da8df4f681a68467178 provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.step_data( |
| 'verify raw dartsdk-macos-arm64-release.zip hash/abfdc3d50f6cf66165767da8df4f681a68467178 provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.post_process(MustRun, 'cloudkms get key codesign_key'), |
| api.step_data('delete previous keychain', retcode=1), |
| find_identity, |
| api.post_process(MustRun, 'codesign dartsdk-macos-x64-release/dart'), |
| api.post_process(MustRun, 'notarize'), |
| api.post_process(MustRun, 'delete keychain'), |
| api.post_process( |
| MustRun, |
| 'gsutil upload signed dartsdk-macos-x64-release.zip hash/abfdc3d50f6cf66165767da8df4f681a68467178' |
| ), |
| api.post_process( |
| MustRun, |
| 'gsutil upload signed dartsdk-macos-arm64-release.zip hash/abfdc3d50f6cf66165767da8df4f681a68467178' |
| ), |
| api.post_process(MustRun, 'register dart/dart-sdk/mac-amd64'), |
| api.post_process(MustRun, 'register dart/dart-sdk/mac-arm64'), |
| api.post_process(MustRun, 'snoop: report_stage (5)'), |
| ) |
| yield api.test( |
| 'sign-failed', |
| api.properties(Sign(version='2.19.5', archs=ARCHS)), |
| api.step_data( |
| 'gsutil check if dartsdk-macos-x64-release.zip ' |
| '2.19.5 is already signed', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if dartsdk-macos-arm64-release.zip ' |
| '2.19.5 is already signed', |
| retcode=1), |
| version_data, |
| api.step_data( |
| 'verify raw dartsdk-macos-x64-release.zip 2.19.5 provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.step_data( |
| 'verify raw dartsdk-macos-arm64-release.zip 2.19.5 provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.post_process(MustRun, 'cloudkms get key codesign_key'), |
| api.step_data('delete previous keychain', retcode=1), |
| find_identity_bad, |
| api.post_process(DropExpectation), |
| status='INFRA_FAILURE', |
| ) |
| yield api.test( |
| 'already', |
| api.properties(Sign(version='2.19.5', archs=ARCHS)), |
| api.post_process(DoesNotRunRE, 'cloudkms get key codesign_key'), |
| api.post_process(DoesNotRunRE, 'notarize'), |
| api.post_process(DoesNotRunRE, 'gsutil upload signed.*'), |
| api.post_process(DoesNotRunRE, 'snoop: report_stage \(5\)'), |
| api.post_process(DoesNotRunRE, 'register dart/dart-sdk/.*'), |
| ) |
| yield api.test( |
| 'dry', |
| api.runtime(is_experimental=True), |
| api.properties(Sign(version='2.19.5', archs=ARCHS)), |
| version_data, |
| api.step_data( |
| 'verify raw dartsdk-macos-x64-release.zip 2.19.5 provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.step_data( |
| 'verify raw dartsdk-macos-arm64-release.zip 2.19.5 provenance', |
| stdout=api.raw_io.output_text('{"allowed": true}')), |
| api.post_process(MustRun, 'cloudkms get key codesign_key'), |
| api.step_data('delete previous keychain', retcode=1), |
| find_identity, |
| api.post_process(MustRun, 'codesign dartsdk-macos-x64-release/dart'), |
| api.post_process(MustRun, 'notarize'), |
| api.post_process(MustRun, 'delete keychain'), |
| api.post_process(DoesNotRunRE, 'gsutil upload signed.*'), |
| api.post_process(DoesNotRunRE, 'register dart/dart-sdk/.*'), |
| api.post_process(MustRun, 'snoop: report_stage (5)'), |
| ) |
| yield api.test( |
| 'provenance-rejected', |
| api.properties(Sign(version='2.19.5', archs=ARCHS)), |
| api.step_data( |
| 'gsutil check if dartsdk-macos-x64-release.zip ' |
| '2.19.5 is already signed', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if dartsdk-macos-arm64-release.zip ' |
| '2.19.5 is already signed', |
| retcode=1), |
| version_data, |
| api.step_data( |
| 'verify raw dartsdk-macos-x64-release.zip 2.19.5 provenance', |
| stdout=api.raw_io.output_text( |
| '{"rejectionMessage": "it\'s not me it\'s you"}')), |
| api.post_process( |
| MustRun, |
| 'raw dartsdk-macos-x64-release.zip 2.19.5 provenance rejected: ' |
| 'it\'s not me it\'s you'), |
| api.post_process(DoesNotRunRE, 'cloudkms get key codesign_key'), |
| api.post_process(DoesNotRunRE, 'notarize'), |
| api.post_process(DoesNotRunRE, 'gsutil upload signed.*'), |
| api.post_process(DoesNotRunRE, 'register dart/dart-sdk/.*'), |
| api.post_process(DoesNotRunRE, 'snoop: report_stage \(5\)'), |
| status='FAILURE', |
| ) |
| yield api.test( |
| 'provenance-outage', |
| api.properties(Sign(version='2.19.5', archs=ARCHS)), |
| api.step_data( |
| 'gsutil check if dartsdk-macos-x64-release.zip ' |
| '2.19.5 is already signed', |
| retcode=1), |
| api.step_data( |
| 'gsutil check if dartsdk-macos-arm64-release.zip ' |
| '2.19.5 is already signed', |
| retcode=1), |
| version_data, |
| api.step_data( |
| 'verify raw dartsdk-macos-x64-release.zip 2.19.5 provenance', |
| stdout=api.raw_io.output_text('internal server error')), |
| api.post_process(MustRun, 'Provenance verification API failed'), |
| api.post_process(DoesNotRunRE, 'cloudkms get key codesign_key'), |
| api.post_process(DoesNotRunRE, 'notarize'), |
| api.post_process(DoesNotRunRE, 'gsutil upload signed.*'), |
| api.post_process(DoesNotRunRE, 'register dart/dart-sdk/.*'), |
| api.post_process(DoesNotRunRE, 'snoop: report_stage \(5\)'), |
| status='INFRA_FAILURE', |
| ) |
