| # Copyright 2022 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| import itertools |
| import json |
| import datetime |
| |
| from recipe_engine.post_process import ( |
| DoesNotRun, |
| DropExpectation, |
| MustRun, |
| StepCommandContains, |
| StepCommandDoesNotContain, |
| StatusFailure, |
| ) |
| |
| PYTHON_VERSION_COMPATIBILITY = 'PY3' |
| |
| DEPS = [ |
| 'dart', |
| 'depot_tools/bot_update', |
| 'depot_tools/depot_tools', |
| 'depot_tools/gclient', |
| 'depot_tools/git', |
| 'depot_tools/gitiles', |
| 'recipe_engine/buildbucket', |
| 'recipe_engine/context', |
| 'recipe_engine/file', |
| 'recipe_engine/json', |
| 'recipe_engine/path', |
| 'recipe_engine/properties', |
| 'recipe_engine/runtime', |
| 'recipe_engine/step', |
| ] |
| |
| # Do not fetch and roll more than this many commits from a source repo. |
| # If this needs to be changed temporarily, consider making it configurable |
| # from the builder properties. |
| FETCH_LIMIT = 1000 |
| |
| COMMITS_JSON = 'commits.json' |
| EXAMPLE_COMMITS_JSON = { |
| 'flutter': 'flutter_hash', |
| 'engine/src/flutter': 'engine_hash', |
| 'engine/src/third_party/dart': 'dart_sdk_hash', |
| } |
| |
| REPOSITORIES = { |
| 'flutter': 'external/github.com/flutter/flutter', |
| 'engine/src/flutter': 'external/github.com/flutter/engine', |
| 'engine/src/third_party/dart': 'sdk', |
| } |
| DART_HOST = 'https://dart.googlesource.com' |
| MONOREPO_URL = '%s/monorepo' % DART_HOST |
| MONOREPO_REF = 'refs/heads/main' |
| |
| |
| def RunSteps(api): |
| with api.context(cwd=api.path['cache'].join('builder')): |
| src_cfg = api.gclient.make_config() |
| soln = src_cfg.solutions.add() |
| soln.name = 'monorepo' |
| soln.url = MONOREPO_URL |
| soln.deps_file = 'DEPS' |
| soln.managed = False |
| soln.custom_vars = {'download_android_deps': False} |
| src_cfg.revisions = {'monorepo': MONOREPO_REF} |
| api.gclient.c = src_cfg |
| api.bot_update.ensure_checkout( |
| ignore_input_commit=True, set_output_commit=False) |
| |
| with api.context(cwd=api.path['cache'].join('builder/monorepo')): |
| commits = read_commits_json(api) |
| commit_lists = { |
| source_path: fetch_commits(api, '%s/%s' % (DART_HOST, repository), |
| commits[source_path]) |
| for source_path, repository in REPOSITORIES.items() |
| } |
| new_commits = linearized_commits(api, commit_lists) |
| api.git('config', '--get', 'user.email') |
| with api.context( |
| cwd=api.path['cache'].join('builder/engine/src/flutter')): |
| api.step('fetch engine commits', [ |
| 'python3', |
| api.depot_tools.root.join('git_cache.py'), 'fetch', '-c', |
| src_cfg.cache_dir |
| ]) |
| with api.context( |
| cwd=api.path['cache'].join('builder/engine/src/third_party/dart')): |
| api.step('fetch dart commits', [ |
| 'python3', |
| api.depot_tools.root.join('git_cache.py'), 'fetch', '-c', |
| src_cfg.cache_dir |
| ]) |
| # Only roll the first 50 linearized commits in a single run. |
| for source_path, commit in new_commits[:50]: |
| commits = create_new_commit(api, commits, source_path, commit) |
| if api.runtime.is_experimental: |
| api.git('push', '--dry-run', MONOREPO_URL, 'HEAD:%s' % MONOREPO_REF) |
| else: |
| api.git('push', MONOREPO_URL, 'HEAD:%s' % MONOREPO_REF) |
| |
| |
| def read_commits_json(api): |
| result = api.json.read( |
| 'read old commits', |
| api.path['cache'].join('builder/monorepo', COMMITS_JSON), |
| step_test_data=lambda: api.json.test_api.output(EXAMPLE_COMMITS_JSON) |
| ).json |
| return result.output |
| |
| |
| def fetch_commits(api, repository, commit_hash): |
| fetched = [] |
| cursor = None |
| limit = 10 |
| while len(fetched) < FETCH_LIMIT: # pragma: no cover |
| new_fetched, cursor = api.gitiles.log( |
| repository, |
| 'main', |
| step_name='Fetch commits from %s' % repository, |
| attempts=5, |
| limit=limit, |
| cursor=cursor) |
| fetched.extend(new_fetched) |
| if any(commit['commit'] == commit_hash for commit in new_fetched): |
| break |
| limit = 100 |
| else: |
| raise api.step.StepFailure('Did not find current commit in repository' |
| ' %s looking back %s commits. Could not find' |
| ' commit %s' % |
| (repository, FETCH_LIMIT, commit_hash)) |
| return list( |
| itertools.takewhile(lambda commit: commit['commit'] != commit_hash, |
| fetched)) |
| |
| |
| def linearized_commits(api, commit_lists): |
| """Iterate backwards through the lists, always choosing the element with |
| |
| the earliest timestamp. This also works if a list has an out-of-order |
| timestamp, which a merge-and-sort would not handle correctly. |
| """ |
| result = [] |
| indexes = { |
| source_path: len(commits) - 1 |
| for source_path, commits in commit_lists.items() |
| } |
| while any(index >= 0 for index in indexes.values()): |
| next_timestamp = None |
| for source_path, index in indexes.items(): |
| if index >= 0 and (not next_timestamp or timestamp( |
| commit_lists[source_path][index]) < next_timestamp): |
| rolled_path = source_path |
| next_timestamp = timestamp(commit_lists[source_path][index]) |
| result.append( |
| (rolled_path, commit_lists[rolled_path][indexes[rolled_path]])) |
| indexes[rolled_path] -= 1 |
| api.step.empty('%s commits to roll' % len(result),) |
| return result |
| |
| |
| def create_new_commit(api, commits, source_path, commit): |
| commit_hash = commit['commit'] |
| commits[source_path] = commit_hash |
| # Use the modified commits.json to create a modified DEPS |
| api.git( |
| '-C', |
| api.path['cache'].join('builder/engine/src/flutter'), |
| 'checkout', |
| commits['engine/src/flutter'], |
| name='check out engine commit') |
| api.git( |
| '-C', |
| api.path['cache'].join('builder/engine/src/third_party/dart'), |
| 'checkout', |
| commits['engine/src/third_party/dart'], |
| name='check out sdk commit') |
| with api.context(cwd=api.path['cache'].join('builder')): |
| api.step('update dependencies', [ |
| api.path['cache'].join( |
| 'builder/engine/src/third_party/dart/tools/sdks/dart-sdk/bin/dart'), |
| api.path['cache'].join( |
| 'builder/monorepo/tools/create_monorepo_deps.dart'), |
| '--dart=%s' % commits['engine/src/third_party/dart'], |
| '--engine=%s' % commits['engine/src/flutter'], |
| '--flutter=%s' % commits['flutter'] |
| ]) |
| # Verify that the modified dependencies can be fetched |
| api.gclient('verify gclient sync with updated dependencies', |
| ['sync', '--nohooks']) |
| |
| url = '%s/%s' % (DART_HOST, REPOSITORIES[source_path]) |
| commit_url = '%s/+/%s' % (url, commit_hash) |
| message = '%s\n%s\n' % (commit['message'], commit_url) |
| commit_author = commit['author'] |
| author = '%s <%s>' % (commit_author['name'], commit_author['email']) |
| author_date = commit_author['time'] |
| commits[source_path] = commit_hash |
| pretty_json = json.dumps( |
| commits, indent=2, separators=(',', ':'), sort_keys=True) |
| api.file.write_text('update commits', |
| api.path['cache'].join('builder/monorepo', |
| COMMITS_JSON), pretty_json) |
| api.git('add', api.path['cache'].join('builder/monorepo', COMMITS_JSON), |
| 'DEPS') |
| api.git('commit', '--author=%s' % author, '--date=%s' % author_date, '-m', |
| message) |
| return commits |
| |
| |
| def timestamp(commit): |
| return datetime.datetime.strptime(commit['committer']['time'], |
| '%a %b %d %H:%M:%S %Y %z').timestamp() |
| |
| |
| def formatted_time(time): |
| return time.strftime('%a %b %d %H:%M:%S %Y %z') |
| |
| |
| def GenTests(api): |
| for experimental in [True, False]: |
| yield api.test( |
| 'experimental' if experimental else 'production', |
| api.properties.generic(repo='https://unused_commit'), |
| api.buildbucket.ci_build( |
| git_ref='unused_branch', |
| git_repo='https://dart.googlesource.com/a/repo.git', |
| revision='unused_deadbeef', |
| experiments=['luci.non_production'] if experimental else [], |
| ), |
| api.runtime(is_experimental=experimental), |
| api.step_data( |
| 'read old commits', |
| api.json.output({ |
| 'flutter': 'existing flutter commit hash', |
| 'engine/src/flutter': 'existing engine commit hash', |
| 'engine/src/third_party/dart': 'existing dart commit hash', |
| })), |
| api.step_data( |
| 'Fetch commits from https://dart.googlesource.com/external/github.com/flutter/flutter', |
| flutter_log(api)), |
| api.step_data( |
| 'Fetch commits from https://dart.googlesource.com/external/github.com/flutter/engine', |
| engine_log(api)), |
| api.step_data('Fetch commits from https://dart.googlesource.com/sdk', |
| first_dart_log(api)), |
| api.step_data( |
| 'Fetch commits from https://dart.googlesource.com/sdk (2)', |
| second_dart_log(api)), |
| api.post_process( |
| StepCommandContains, 'git commit (12)', |
| '--author=fake_dart_first <fake_dart_first@fake_2.email.com>'), |
| api.post_process(MustRun, 'git commit (15)'), |
| api.post_process(DoesNotRun, 'git commit (16)'), |
| api.post_process( |
| StepCommandContains if experimental else StepCommandDoesNotContain, |
| 'git push', '--dry-run'), |
| ) |
| |
| flutter_fetches = [ |
| api.step_data( |
| 'Fetch commits from https://dart.googlesource.com/external/github.com/flutter/flutter (%s)' |
| % i, long_flutter_log(api)) for i in range(2, 35) |
| ] |
| |
| yield api.test( |
| 'commit-not-found', api.properties.generic(repo='https://unused_commit'), |
| api.buildbucket.ci_build( |
| git_ref='unused_branch', |
| git_repo='https://dart.googlesource.com/a/repo.git', |
| revision='unused_deadbeef'), |
| api.step_data( |
| 'read old commits', |
| api.json.output({ |
| 'flutter': 'existing flutter commit hash', |
| 'engine/src/flutter': 'existing engine commit hash', |
| 'engine/src/third_party/dart': 'existing dart commit hash', |
| })), |
| api.step_data( |
| 'Fetch commits from https://dart.googlesource.com/external/github.com/flutter/flutter', |
| long_flutter_log(api)), |
| api.post_process( |
| DoesNotRun, |
| 'Fetch commits from https://dart.googlesource.com/external/github.com/flutter/engine' |
| ), api.post_process(StatusFailure), api.post_process(DropExpectation), |
| *flutter_fetches) |
| |
| |
| def modified_commit_times(api, |
| step_data, |
| time, |
| offset=0, |
| interval=1, |
| commit_hash=None, |
| commit_hash_index=None): |
| log = api.json.loads(step_data.unwrap_placeholder().data) |
| if commit_hash: |
| log['log'][commit_hash_index]['commit'] = commit_hash |
| for i, commit in enumerate(log['log']): |
| delta = datetime.timedelta(minutes=offset + i * interval) |
| commit['committer']['time'] = formatted_time(time - delta) |
| return api.json.output(log) |
| |
| |
| def end_time(): |
| try: |
| return datetime.datetime.strptime('Thu Mar 31 09:09:00 2022 +0200', |
| '%a %b %d %H:%M:%S %Y %z') |
| # Added for python2 compatibility required to run recipe test command. |
| except: # pragma: no cover |
| return datetime.datetime.strptime('Thu Mar 31 09:09:00 2022', |
| '%a %b %d %H:%M:%S %Y') |
| |
| |
| def flutter_log(api): |
| step_data = api.gitiles.make_log_test_data('flutter', 5, 'flutter_page_token') |
| return modified_commit_times( |
| api, |
| step_data, |
| end_time(), |
| interval=2, |
| commit_hash='existing flutter commit hash', |
| commit_hash_index=4) |
| |
| |
| def long_flutter_log(api): |
| return api.gitiles.make_log_test_data('flutter', 30, 'flutter_page_token') |
| |
| |
| def engine_log(api): |
| step_data = api.gitiles.make_log_test_data('engine', 5, 'engine_page_token') |
| return modified_commit_times( |
| api, |
| step_data, |
| end_time(), |
| offset=1, |
| interval=2, |
| commit_hash='existing engine commit hash', |
| commit_hash_index=2) |
| |
| |
| def first_dart_log(api): |
| step_data = api.gitiles.make_log_test_data('dart_first', 3, |
| 'dart_first_page_token') |
| return modified_commit_times(api, step_data, end_time(), offset=1, interval=2) |
| |
| |
| def second_dart_log(api): |
| step_data = api.gitiles.make_log_test_data('dart_second', 10, |
| 'dart_second_page_token') |
| return modified_commit_times( |
| api, |
| step_data, |
| end_time(), |
| offset=1, |
| interval=2, |
| commit_hash='existing dart commit hash', |
| commit_hash_index=6) |
