| # Copyright 2022 The Dart Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import re |
| |
| from recipe_engine import post_process |
| |
| from PB.recipes.dart.release.merge import Merge |
| |
| DEPS = [ |
| 'dart', |
| 'depot_tools/bot_update', |
| 'depot_tools/gclient', |
| 'depot_tools/git', |
| 'depot_tools/gitiles', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/raw_io', |
| 'recipe_engine/runtime', |
| 'recipe_engine/step', |
| 'recipe_engine/time', |
| ] |
| MESSAGE_TEMPLATE = '''Version %s |
| |
| Merge %s into %s |
| ''' |
| PROPERTIES = Merge |
| REPO_URL = 'https://dart.googlesource.com/sdk' |
| SUPPORTED_FIELDS = { |
| 'CHANNEL', 'MAJOR', 'MINOR', 'PATCH', 'PRERELEASE', 'PRERELEASE_PATCH' |
| } |
| VERSION_PATH = 'tools/VERSION' |
| CHANGELOG_PATH = 'CHANGELOG.md' |
| SOURCE_BRANCHES = { |
| 'dev': 'main', # dev is merged from lkgr, but all lkgr commits are on main |
| 'beta': 'dev', |
| 'stable': 'beta', |
| } |
| |
| PYTHON_VERSION_COMPATIBILITY = "PY3" |
| |
| |
| def RunSteps(api, properties): |
| # from_ref can be merged to to_ref, if from_ref's commit exists on the branch |
| # that is supposed to roll into the to_ref branch. E.g. any dev release from |
| # the dev branch can be rolled into beta. |
| from_ref = properties.from_ref |
| assert from_ref, 'the recipe requires a from_ref' |
| to_ref = properties.to_ref |
| assert to_ref, 'the recipe requires a to_ref' |
| assert to_ref.startswith('refs/heads/'), 'to_ref must start with refs/heads/' |
| target_branch = to_ref.replace('refs/heads/', '') |
| assert target_branch in SOURCE_BRANCHES, 'target branch must be allowed' |
| source_branch = SOURCE_BRANCHES[target_branch] |
| if target_branch != 'dev': |
| assert from_ref == f'refs/heads/{source_branch}' or ( |
| from_ref.startswith('refs/tags/') and |
| from_ref.endswith(f'.{source_branch}') |
| ), f'must merge from {source_branch} branch or {source_branch} tag' |
| |
| api.git.checkout( |
| REPO_URL, |
| ref=to_ref, |
| submodules=False, |
| submodule_update_recursive=False, |
| ) |
| with api.context(cwd=api.path['checkout']): |
| api.git('config', 'user.name', 'Dart CI', name='configure user.name') |
| api.git( |
| 'config', |
| 'user.email', |
| api.buildbucket.swarming_task_service_account, |
| name='configure user.email') |
| |
| api.git('checkout', target_branch, name=f'checkout {target_branch}') |
| api.git('fetch', 'origin', from_ref, name=f'fetch {from_ref}') |
| # rev-list -n1 is used to dereference annotated tags to the real commit. |
| commit_hash = api.git( |
| 'rev-list', |
| '-n1', |
| 'FETCH_HEAD', |
| name='get commit to merge', |
| stdout=api.raw_io.output_text(add_output_log=True)).stdout.rstrip() |
| api.git('fetch', 'origin', source_branch, name=f'fetch {source_branch}') |
| result = api.git( |
| 'merge-base', |
| '--is-ancestor', |
| commit_hash, |
| f'origin/{source_branch}', |
| name=f'verify commit is on source branch {source_branch}', |
| ) |
| result = api.git( |
| 'merge-base', |
| '--is-ancestor', |
| commit_hash, |
| 'HEAD', |
| name='check if commit has been merged before', |
| ok_ret=[0, 1], # merge-base returns 1 if the commit is not an ancestor. |
| ) |
| if result.exc_result.retcode == 0: |
| # The commit has already been pushed before. |
| return |
| |
| target_version_text = _read_version_file(api, 'read target version') |
| target_version = api.dart.Version(text=target_version_text) |
| |
| api.git( |
| 'merge', |
| '--no-commit', |
| '--no-ff', |
| commit_hash, |
| name=f'merge {commit_hash} to {target_branch}', |
| ok_ret='any') |
| api.git('reset', commit_hash, '.', name=f'reset to {commit_hash}') |
| api.git('restore', '.', name='restore to index') |
| |
| # Determine the new version number and update the version file. |
| source_version_text = _read_version_file(api, 'read source version') |
| source_version = api.dart.Version(text=source_version_text) |
| source_version_string = commit_hash if target_branch == 'dev' else str( |
| source_version) |
| version = _increment_version(api, source_version, target_version, |
| target_branch) |
| source_version.set_version(version) |
| _write_version_file(api, source_version.text) |
| |
| # Stable releases use the latest CHANGELOG.md from main to simplify the |
| # workflow to ensuring the finished documentation is in one place. The |
| # sections for later releases are automatically cut away. beta and dev |
| # releases instead use the changelog from when they branched, as it is more |
| # accurate about the contents than features later landed on main. |
| if target_branch == 'stable': |
| changelog = _fetch_changelog(api, version) |
| api.file.write_text('write CHANGELOG.md', |
| api.path['checkout'].join(CHANGELOG_PATH), changelog) |
| api.git('add', CHANGELOG_PATH) |
| |
| message = MESSAGE_TEMPLATE % (version, source_version_string, target_branch) |
| api.git('commit', '--all', f'--message={message}') |
| |
| # Tags for beta and stable are made upon publishing instead of merging. |
| if target_branch == 'dev': |
| api.git('tag', '--annotate', f'--message={version}', version) |
| push_args = ['push', '--atomic'] |
| if (api.buildbucket.builder_name.endswith('-try') or |
| api.runtime.is_experimental): |
| push_args.append('--dry-run') |
| push_args.append('https://dart.googlesource.com/sdk.git') |
| push_args.append(to_ref) |
| if target_branch == 'dev': |
| push_args.append(f'refs/tags/{version}') |
| try: |
| api.git(*push_args, name=f'push to {to_ref}') |
| except api.step.InfraFailure: |
| # If the step fails, it's likely because of GoB replication delay. Wait |
| # 30s and then try again. |
| api.time.sleep(30) |
| api.git(*push_args, name=f'retry push to {to_ref}') |
| |
| |
| def _increment_version(api, from_version, to_version, channel): |
| assert from_version > to_version, ( |
| f'version must be upgraded: {from_version} -> {to_version}') |
| |
| bumped = False |
| for field in ['MAJOR', 'MINOR', 'PATCH']: |
| if to_version.fields[field] != from_version.fields[field]: |
| bumped = True |
| to_version.fields[field] = from_version.fields[field] |
| |
| # dev PRERELEASE increments reset on a version bump, PRERELEASE_PATCH is 0 |
| if channel == 'dev': |
| if bumped: |
| to_version.fields['PRERELEASE'] = '0' |
| else: |
| to_version.fields['PRERELEASE'] = str( |
| int(to_version.fields['PRERELEASE']) + 1) |
| to_version.fields['PRERELEASE_PATCH'] = '0' |
| |
| # beta PRERELEASE_PATCH is always 1 after a merge |
| elif channel == 'beta': |
| to_version.fields['PRERELEASE'] = from_version.fields['PRERELEASE'] |
| to_version.fields['PRERELEASE_PATCH'] = '1' |
| |
| # stable PRERELEASE and PRERELEASE_PATCH are both 0 |
| elif channel == 'stable': |
| to_version.fields['PRERELEASE'] = '0' |
| to_version.fields['PRERELEASE_PATCH'] = '0' |
| |
| to_version.fields['CHANNEL'] = channel |
| |
| return str(to_version) |
| |
| |
| def _read_version_file(api, step_name): |
| return api.file.read_text(step_name, api.path['checkout'].join(VERSION_PATH)) |
| |
| |
| def _write_version_file(api, version_file_text): |
| api.file.write_text('write new version file', |
| api.path['checkout'].join(VERSION_PATH), |
| version_file_text) |
| |
| |
| def _fetch_changelog(api, version): |
| api.git('fetch', 'origin', 'main', '--progress', name='git fetch origin main') |
| result = api.git( |
| 'show', |
| f'origin/main:{CHANGELOG_PATH}', |
| name='read CHANGELOG.md from main', |
| stdout=api.raw_io.output()) |
| changelog = result.stdout.decode('utf-8') |
| found = False |
| lines = [] |
| for line in changelog.splitlines(): |
| if line == f'## {version}' or line.startswith(f'## {version} - '): |
| found = True |
| if found: |
| lines.append(line) |
| assert found, f'CHANGELOG.md on main must contain version {version}' |
| return '\n'.join(lines) + '\n' |
| |
| |
| def GenTests(api): |
| buildbucket_build = api.buildbucket.ci_build_message() |
| api.buildbucket.update_backend_service_account(buildbucket_build, |
| 'service@example.com') |
| |
| assertion_error = ( |
| api.post_process(post_process.DoesNotRunRE, '(push|tag).*'), |
| api.expect_exception('AssertionError'), |
| api.post_process(post_process.StatusException), |
| api.post_process(post_process.DropExpectation), |
| ) |
| |
| yield api.test('no-from-ref-no-push', |
| api.buildbucket.build(buildbucket_build), *assertion_error) |
| |
| input_properties = api.properties( |
| Merge(from_ref='refs/heads/lkgr', to_ref='refs/heads/dev')) |
| from_ref_commit = api.step_data( |
| 'get commit to merge', |
| stdout=api.raw_io.output_text( |
| '1f8ac10f23c5b5bc1167bda84b833e5c057a77d2\n')) |
| from_ref_version_file = api.step_data( |
| 'read source version', |
| api.file.read_text('''# Updated Comment |
| CHANNEL main |
| MAJOR 4 |
| MINOR 1 |
| PATCH 2 |
| PRERELEASE 0 |
| PRERELEASE_PATCH 0 |
| UNKNOWN_FIELD cde |
| ''')) |
| to_ref_version_file = api.step_data( |
| 'read target version', |
| api.file.read_text('''# Comment |
| CHANNEL dev |
| MAJOR 4 |
| MINOR 1 |
| PATCH 2 |
| PRERELEASE 34 |
| PRERELEASE_PATCH 25 |
| UNKNOWN_FIELD feg |
| ''')) |
| |
| yield api.test( |
| 'no-push-if-already-merged', |
| input_properties, |
| from_ref_commit, |
| api.buildbucket.build(buildbucket_build), |
| api.post_process(post_process.DoesNotRunRE, '(push|tag).*'), |
| api.post_process(post_process.StatusSuccess), |
| ) |
| |
| expected_version_file = api.post_process( |
| post_process.LogEquals, 'write new version file', 'VERSION', |
| '''# Updated Comment |
| CHANNEL dev |
| MAJOR 4 |
| MINOR 1 |
| PATCH 2 |
| PRERELEASE 35 |
| PRERELEASE_PATCH 0 |
| UNKNOWN_FIELD cde''') |
| unmerged = api.step_data('check if commit has been merged before', retcode=1) |
| yield api.test( |
| 'push', |
| input_properties, |
| from_ref_commit, |
| from_ref_version_file, |
| to_ref_version_file, |
| unmerged, |
| expected_version_file, |
| api.buildbucket.build(buildbucket_build), |
| api.post_process(post_process.MustRun, 'push to refs/heads/dev'), |
| api.post_process(post_process.StepCommandContains, |
| 'push to refs/heads/dev', [ |
| '--atomic', |
| 'https://dart.googlesource.com/sdk.git', |
| 'refs/heads/dev', |
| 'refs/tags/4.1.2-35.0.dev', |
| ]), |
| api.post_process(post_process.StatusSuccess), |
| ) |
| yield api.test( |
| 'push-retry', |
| input_properties, |
| from_ref_commit, |
| from_ref_version_file, |
| to_ref_version_file, |
| unmerged, |
| expected_version_file, |
| api.buildbucket.build(buildbucket_build), |
| api.post_process(post_process.MustRun, 'push to refs/heads/dev'), |
| api.step_data('push to refs/heads/dev', retcode=1), |
| api.post_process(post_process.MustRun, 'retry push to refs/heads/dev'), |
| api.post_process(post_process.StepCommandContains, |
| 'retry push to refs/heads/dev', [ |
| '--atomic', |
| 'https://dart.googlesource.com/sdk.git', |
| 'refs/heads/dev', |
| 'refs/tags/4.1.2-35.0.dev', |
| ]), |
| api.post_process(post_process.StatusSuccess), |
| api.post_process(post_process.Filter().include_re('(retry )?push')), |
| ) |
| yield api.test( |
| 'dry-run', |
| api.runtime(is_experimental=True), |
| input_properties, |
| from_ref_commit, |
| from_ref_version_file, |
| to_ref_version_file, |
| unmerged, |
| expected_version_file, |
| api.buildbucket.build(buildbucket_build), |
| api.post_process(post_process.MustRun, 'push to refs/heads/dev'), |
| api.post_process(post_process.StepCommandContains, |
| 'push to refs/heads/dev', ['--atomic', '--dry-run']), |
| api.post_process(post_process.StatusSuccess), |
| api.post_process(post_process.DropExpectation), |
| ) |
| |
| from_ref_version_file = api.step_data( |
| 'read source version', |
| api.file.read_text('''# Updated Comment |
| CHANNEL main |
| MAJOR 5 |
| MINOR 2 |
| PATCH 10 |
| PRERELEASE 0 |
| PRERELEASE_PATCH 0 |
| UNKNOWN_FIELD cde |
| ''')) |
| expected_version_file = api.post_process( |
| post_process.LogEquals, 'write new version file', 'VERSION', |
| '''# Updated Comment |
| CHANNEL dev |
| MAJOR 5 |
| MINOR 2 |
| PATCH 10 |
| PRERELEASE 0 |
| PRERELEASE_PATCH 0 |
| UNKNOWN_FIELD cde''') |
| yield api.test( |
| 'push-version-mismatch', |
| input_properties, |
| from_ref_commit, |
| from_ref_version_file, |
| to_ref_version_file, |
| unmerged, |
| expected_version_file, |
| api.buildbucket.build(buildbucket_build), |
| api.post_process(post_process.MustRun, 'push to refs/heads/dev'), |
| api.post_process(post_process.StepCommandContains, |
| 'push to refs/heads/dev', [ |
| '--atomic', |
| 'https://dart.googlesource.com/sdk.git', |
| 'refs/heads/dev', |
| 'refs/tags/5.2.10-0.0.dev', |
| ]), |
| api.post_process(post_process.StatusSuccess), |
| api.post_process(post_process.DropExpectation), |
| ) |
| from_ref_version_file = api.step_data( |
| 'read source version', |
| api.file.read_text('''# Comment |
| CHANNEL main |
| MAJOR 4 |
| MINOR 0 |
| PATCH 2 |
| PRERELEASE 0 |
| PRERELEASE_PATCH 0 |
| UNKNOWN_FIELD cde |
| ''')) |
| yield api.test('no-push-version-downgrade', input_properties, from_ref_commit, |
| from_ref_version_file, to_ref_version_file, unmerged, |
| api.buildbucket.build(buildbucket_build), *assertion_error) |
| |
| input_properties = api.properties( |
| Merge(from_ref='refs/heads/dev', to_ref='refs/heads/stable')) |
| yield api.test('forbidden-merge', input_properties, |
| api.buildbucket.build(buildbucket_build), *assertion_error) |
| |
| input_properties = api.properties( |
| Merge(from_ref='refs/heads/dev', to_ref='refs/heads/beta')) |
| from_ref_version_file = api.step_data( |
| 'read source version', |
| api.file.read_text('''# Updated Comment |
| CHANNEL dev |
| MAJOR 2 |
| MINOR 18 |
| PATCH 0 |
| PRERELEASE 271 |
| PRERELEASE_PATCH 0 |
| ''')) |
| to_ref_version_file = api.step_data( |
| 'read target version', |
| api.file.read_text('''# Comment |
| CHANNEL beta |
| MAJOR 2 |
| MINOR 18 |
| PATCH 0 |
| PRERELEASE 165 |
| PRERELEASE_PATCH 1 |
| ''')) |
| expected_version_file = api.post_process( |
| post_process.LogEquals, 'write new version file', 'VERSION', |
| '''# Updated Comment |
| CHANNEL beta |
| MAJOR 2 |
| MINOR 18 |
| PATCH 0 |
| PRERELEASE 271 |
| PRERELEASE_PATCH 1''') |
| yield api.test( |
| 'beta', |
| input_properties, |
| from_ref_commit, |
| from_ref_version_file, |
| to_ref_version_file, |
| unmerged, |
| expected_version_file, |
| api.buildbucket.build(buildbucket_build), |
| api.post_process(post_process.MustRun, 'push to refs/heads/beta'), |
| api.post_process(post_process.StepCommandContains, |
| 'push to refs/heads/beta', [ |
| '--atomic', |
| 'https://dart.googlesource.com/sdk.git', |
| 'refs/heads/beta', |
| ]), |
| api.post_process(post_process.StatusSuccess), |
| ) |
| |
| input_properties = api.properties( |
| Merge(from_ref='refs/heads/beta', to_ref='refs/heads/stable')) |
| from_ref_version_file = api.step_data( |
| 'read source version', |
| api.file.read_text('''# Updated Comment |
| CHANNEL beta |
| MAJOR 2 |
| MINOR 18 |
| PATCH 0 |
| PRERELEASE 271 |
| PRERELEASE_PATCH 8 |
| ''')) |
| to_ref_version_file = api.step_data( |
| 'read target version', |
| api.file.read_text('''# Comment |
| CHANNEL stable |
| MAJOR 2 |
| MINOR 17 |
| PATCH 7 |
| PRERELEASE 0 |
| PRERELEASE_PATCH 0 |
| ''')) |
| expected_version_file = api.post_process( |
| post_process.LogEquals, 'write new version file', 'VERSION', |
| '''# Updated Comment |
| CHANNEL stable |
| MAJOR 2 |
| MINOR 18 |
| PATCH 0 |
| PRERELEASE 0 |
| PRERELEASE_PATCH 0''') |
| main_changelog = api.step_data( |
| 'read CHANGELOG.md from main', |
| stdout=api.raw_io.output('''## 2.19.0 |
| |
| * Exciting upcoming feature. |
| |
| ## 2.18.0 |
| |
| * Neat stable feature. |
| ''')) |
| expected_changelog = api.post_process( |
| post_process.LogEquals, 'write CHANGELOG.md', 'CHANGELOG.md', '''## 2.18.0 |
| |
| * Neat stable feature.''') |
| yield api.test( |
| 'stable', |
| input_properties, |
| from_ref_commit, |
| from_ref_version_file, |
| to_ref_version_file, |
| unmerged, |
| expected_version_file, |
| main_changelog, |
| expected_changelog, |
| api.buildbucket.build(buildbucket_build), |
| api.post_process(post_process.MustRun, 'push to refs/heads/stable'), |
| api.post_process(post_process.StepCommandContains, |
| 'push to refs/heads/stable', [ |
| '--atomic', |
| 'https://dart.googlesource.com/sdk.git', |
| 'refs/heads/stable', |
| ]), |
| api.post_process(post_process.StatusSuccess), |
| ) |
| main_changelog = api.step_data( |
| 'read CHANGELOG.md from main', |
| stdout=api.raw_io.output('''## 2.17.0 |
| |
| * Aged to perfection. |
| ''')) |
| yield api.test('missing-changelog', input_properties, from_ref_commit, |
| from_ref_version_file, to_ref_version_file, unmerged, |
| expected_version_file, main_changelog, *assertion_error, |
| api.buildbucket.build(buildbucket_build)) |
