| #!/usr/bin/env vpython |
| # Copyright 2017 The Chromium Authors. All rights reserved. |
| # Use of this source code is governed by a BSD-style license that can be |
| # found in the LICENSE file. |
| |
| """Tool to move Blink source from third_party/WebKit to third_party/blink. |
| |
| See https://docs.google.com/document/d/1l3aPv1Wx__SpRkdOhvJz8ciEGigNT3wFKv78XiuW0Tw/edit?usp=sharing#heading=h.o225wrxp242h |
| for the details. |
| """ |
| |
| import argparse |
| import logging |
| import os |
| import re |
| import sys |
| from functools import partial |
| |
| # Without abspath(), PathFinder can't find chromium_base correctly. |
| sys.path.append(os.path.abspath( |
| os.path.join(os.path.dirname(__file__), '..', '..', '..', |
| 'third_party', 'WebKit', 'Tools', 'Scripts'))) |
| from blinkpy.common.name_style_converter import NameStyleConverter |
| from plan_blink_move import plan_blink_move |
| from plan_blink_move import relative_dest |
| from webkitpy.common.checkout.git import Git |
| from webkitpy.common.path_finder import get_chromium_src_dir |
| from webkitpy.common.path_finder import get_scripts_dir |
| from webkitpy.common.system.executive import Executive |
| from webkitpy.common.system.filesystem import FileSystem |
| |
| _log = logging.getLogger('move_blink_source') |
| |
| |
| class FileType(object): |
| NONE = 0 |
| BUILD = 1 |
| BLINK_BUILD = 2 |
| OWNERS = 3 |
| DEPS = 4 |
| MOJOM = 5 |
| TYPEMAP = 6 |
| BLINK_BUILD_PY = 7 |
| LAYOUT_TESTS_WITH_MOJOM = 8 |
| |
| @staticmethod |
| def detect(path): |
| slash_dir, basename = os.path.split(path) |
| slash_dir = slash_dir.replace(os.path.sep, '/') |
| if basename == 'DEPS': |
| return FileType.DEPS |
| if basename == 'OWNERS': |
| return FileType.OWNERS |
| if basename.endswith('.mojom'): |
| return FileType.MOJOM |
| if basename.endswith('.typemap'): |
| return FileType.TYPEMAP |
| if basename.endswith('.py') and 'third_party/WebKit/Source/build' in slash_dir: |
| return FileType.BLINK_BUILD_PY |
| if basename.endswith(('.gn', '.gni')): |
| if 'third_party/WebKit' in path or 'third_party/blink' in slash_dir: |
| return FileType.BLINK_BUILD |
| if 'third_party' in slash_dir: |
| return FileType.NONE |
| return FileType.BUILD |
| if basename.endswith('.html') and re.search( |
| r'third_party/WebKit/LayoutTests/(geolocation-api|installedapp|' + |
| r'media/mediasession|payments|presentation|webshare)', slash_dir): |
| return FileType.LAYOUT_TESTS_WITH_MOJOM |
| return FileType.NONE |
| |
| |
| class MoveBlinkSource(object): |
| |
| def __init__(self, fs, options, repo_root): |
| self._fs = fs |
| self._options = options |
| _log.debug(options) |
| self._repo_root = repo_root |
| |
| # The following fields are initialized in _create_basename_maps. |
| self._basename_map = None |
| self._basename_re = None |
| self._idl_generated_impl_headers = None |
| # _checked_in_header_re is used to distinguish checked-in header files |
| # and generated header files. |
| self._checked_in_header_re = None |
| |
| self._updated_files = [] |
| |
| def update(self): |
| _log.info('Planning renaming ...') |
| file_pairs = plan_blink_move(self._fs, []) |
| _log.info('Will move %d files', len(file_pairs)) |
| |
| self._create_basename_maps(file_pairs) |
| dirs = self._update_file_content() |
| |
| # Updates #includes in files in directories with updated DEPS + |
| # third_party/WebKit/{Source,common,public}. |
| self._append_unless_upper_dir_exists(dirs, self._fs.join(self._repo_root, 'third_party', 'WebKit', 'Source')) |
| self._append_unless_upper_dir_exists(dirs, self._fs.join(self._repo_root, 'third_party', 'WebKit', 'common')) |
| self._append_unless_upper_dir_exists(dirs, self._fs.join(self._repo_root, 'third_party', 'WebKit', 'public')) |
| self._append_unless_upper_dir_exists(dirs, self._fs.join(self._repo_root, 'mojo', 'public', 'tools', |
| 'bindings', 'generators', 'cpp_templates')) |
| self._update_cpp_includes_in_directories(dirs) |
| |
| # Content update for individual files. |
| # The following is a list of tuples. |
| # Tuple: (<file path relative to repo root>, [replacement commands]) |
| # Command: a callable object, or |
| # a tuple of (<original string>, <new string>). |
| file_replacement_list = [ |
| ('DEPS', |
| [('src/third_party/WebKit/Source/devtools', |
| 'src/third_party/blink/renderer/devtools')]), |
| ('WATCHLISTS', |
| [('third_party/WebKit/Source', 'third_party/blink/renderer'), |
| ('third_party/WebKit/public', 'third_party/blink/renderer/public')]), |
| ('build/check_gn_headers_whitelist.txt', |
| [('third_party/WebKit/Source', 'third_party/blink/renderer'), |
| ('third_party/WebKit/public', 'third_party/blink/renderer/public'), |
| self._update_basename]), |
| ('testing/buildbot/gn_isolate_map.pyl', |
| [('third_party/WebKit/Source', 'third_party/blink/renderer')]), |
| ('third_party/WebKit/Source/BUILD.gn', |
| [('$root_gen_dir/third_party/WebKit', |
| '$root_gen_dir/third_party/blink/renderer')]), |
| ('third_party/WebKit/Source/config.gni', |
| [('snake_case_source_files = false', |
| 'snake_case_source_files = true')]), |
| ('third_party/WebKit/Source/core/css/CSSProperties.json5', |
| [self._update_basename]), |
| ('third_party/WebKit/Source/core/css/ComputedStyleExtraFields.json5', |
| [self._update_basename]), |
| ('third_party/WebKit/Source/core/css/ComputedStyleFieldAliases.json5', |
| [self._update_basename]), |
| ('third_party/WebKit/Source/core/html/parser/create-html-entity-table', |
| [self._update_basename]), |
| ('third_party/WebKit/Source/core/inspector/inspector_protocol_config.json', |
| [self._update_basename]), |
| ('third_party/WebKit/Source/core/probe/CoreProbes.json5', |
| [self._update_basename]), |
| ('third_party/WebKit/Source/core/testing/InternalSettings.h', |
| [('InternalSettingsGenerated.h', 'internal_settings_generated.h')]), |
| ('third_party/WebKit/Source/core/testing/Internals.cpp', |
| [('InternalRuntimeFlags.h', 'internal_runtime_flags.h')]), |
| ('third_party/WebKit/Source/platform/probe/PlatformProbes.json5', |
| [self._update_basename]), |
| ('third_party/WebKit/public/BUILD.gn', |
| [('$root_gen_dir/third_party/WebKit', |
| '$root_gen_dir/third_party/blink/renderer')]), |
| ('third_party/WebKit/public/blink_resources.grd', |
| [('../Source/', '../')]), |
| ('tools/android/eclipse/.classpath', |
| [('third_party/WebKit/public', 'third_party/blink/renderer/public')]), |
| ('tools/android/loading/cloud/backend/deploy.sh', |
| [('third_party/WebKit/Source', 'third_party/blink/renderer')]), |
| ('tools/android/loading/emulation_unittest.py', |
| [('third_party/WebKit/Source', 'third_party/blink/renderer')]), |
| ('tools/android/loading/options.py', |
| [('third_party/WebKit/Source', 'third_party/blink/renderer')]), |
| ('tools/android/loading/request_track.py', |
| [('third_party/WebKit/Source', 'third_party/blink/renderer')]), |
| ('tools/gritsettings/resource_ids', |
| [('third_party/WebKit/public', 'third_party/blink/renderer/public'), |
| ('third_party/WebKit/Source', 'third_party/blink/renderer')]), |
| ('tools/metrics/actions/extract_actions.py', |
| [('third_party/WebKit/Source', 'third_party/blink/renderer')]), |
| ('tools/metrics/histograms/update_editor_commands.py', |
| [('third_party/WebKit/Source/core/editing/EditorCommand.cpp', |
| 'third_party/blink/renderer/core/editing/editor_command.cc')]), |
| ('tools/metrics/histograms/update_use_counter_css.py', |
| [('third_party/WebKit/Source/core/frame/UseCounter.cpp', |
| 'third_party/blink/renderer/core/frame/use_counter.cc')]), |
| ('tools/metrics/histograms/update_use_counter_feature_enum.py', |
| [('third_party/WebKit/public', 'third_party/blink/renderer/public')]), |
| ] |
| for file_path, replacement_list in file_replacement_list: |
| self._update_single_file_content(file_path, replacement_list, should_write=self._options.run) |
| |
| if self._options.run: |
| _log.info('Formatting updated %d files ...', len(self._updated_files)) |
| git = Git(cwd=self._repo_root) |
| # |git cl format| can't handle too many files at once. |
| while len(self._updated_files) > 0: |
| end_index = 100 |
| if end_index > len(self._updated_files): |
| end_index = len(self._updated_files) |
| git.run(['cl', 'format'] + self._updated_files[:end_index]) |
| self._updated_files = self._updated_files[end_index:] |
| |
| _log.info('Make a local commit ...') |
| git.commit_locally_with_message("""The Great Blink mv for source files, part 1. |
| |
| Update file contents without moving files. |
| |
| Bug: 768828 |
| """) |
| |
| def move(self): |
| _log.info('Planning renaming ...') |
| file_pairs = plan_blink_move(self._fs, []) |
| _log.info('Will move %d files', len(file_pairs)) |
| |
| git = Git(cwd=self._repo_root) |
| for i, (src, dest) in enumerate(file_pairs): |
| src_from_repo = self._fs.join('third_party', 'WebKit', src) |
| dest_from_repo = self._fs.join('third_party', 'blink', dest) |
| self._fs.maybe_make_directory(self._repo_root, 'third_party', 'blink', self._fs.dirname(dest)) |
| if self._options.run_git: |
| if git.exists(src_from_repo): |
| git.move(src_from_repo, dest_from_repo) |
| _log.info('[%d/%d] Git moved %s', i + 1, len(file_pairs), src) |
| else: |
| _log.info('%s is not in the repository', src) |
| else: |
| self._fs.move(self._fs.join(self._repo_root, src_from_repo), |
| self._fs.join(self._repo_root, dest_from_repo)) |
| _log.info('[%d/%d] Moved %s', i + 1, len(file_pairs), src) |
| self._update_single_file_content( |
| 'build/get_landmines.py', |
| [('\ndef main', ' print \'The Great Blink mv for source files (crbug.com/768828)\'\n\ndef main')]) |
| |
| _log.info('Run run-bindings-tests ...') |
| Executive().run_command(['python', |
| self._fs.join(get_scripts_dir(), 'run-bindings-tests'), |
| '--reset-results'], |
| cwd=self._repo_root) |
| |
| if self._options.run_git: |
| _log.info('Make a local commit ...') |
| git.commit_locally_with_message("""The Great Blink mv for source files, part 2. |
| |
| Move and rename files. |
| |
| Bug: 768828 |
| """) |
| |
| def _create_basename_maps(self, file_pairs): |
| basename_map = {} |
| # Generated inspector/protocol/* contains a lot of names duplicated with |
| # checked-in core files. We don't want to rename them, and don't want to |
| # replace them in BUILD.gn and #include accidentally. |
| pattern = r'(?<!inspector/protocol/)\b(' |
| idl_headers = set() |
| header_pattern = r'(?<!inspector/protocol/)\b(' |
| for source, dest in file_pairs: |
| _, source_base = self._fs.split(source) |
| _, dest_base = self._fs.split(dest) |
| # ConditionalFeaturesForCore.h in bindings/tests/results/modules/ |
| # confuses generated/checked-in detection in _replace_include_path(). |
| if 'bindings/tests' in source.replace('\\', '/'): |
| continue |
| if source_base.endswith('.h'): |
| header_pattern += re.escape(source_base) + '|' |
| if source_base == dest_base: |
| continue |
| basename_map[source_base] = dest_base |
| pattern += re.escape(source_base) + '|' |
| # IDL sometimes generates implementation files as well as |
| # binding files. We'd like to update #includes for such files. |
| if source_base.endswith('.idl'): |
| source_header = source_base.replace('.idl', '.h') |
| basename_map[source_header] = dest_base.replace('.idl', '.h') |
| pattern += re.escape(source_header) + '|' |
| idl_headers.add(source_header) |
| _log.info('Rename %d files for snake_case', len(basename_map)) |
| self._basename_map = basename_map |
| self._basename_re = re.compile(pattern[0:len(pattern) - 1] + ')(?=["\']|$)') |
| self._idl_generated_impl_headers = idl_headers |
| self._checked_in_header_re = re.compile(header_pattern[0:len(header_pattern) - 1] + ')$') |
| |
| def _shorten_path(self, path): |
| if path.startswith(self._repo_root): |
| return path[len(self._repo_root) + 1:] |
| return path |
| |
| @staticmethod |
| def _filter_file(fs, dirname, basename): |
| return FileType.detect(fs.join(dirname, basename)) != FileType.NONE |
| |
| def _update_build(self, content): |
| content = content.replace('//third_party/WebKit/Source', '//third_party/blink/renderer') |
| content = content.replace('//third_party/WebKit/common', '//third_party/blink/common') |
| content = content.replace('//third_party/WebKit/public', '//third_party/blink/renderer/public') |
| # export_header_blink exists outside of Blink too. |
| content = content.replace('export_header_blink = "third_party/WebKit/public/platform/WebCommon.h"', |
| 'export_header_blink = "third_party/blink/renderer/public/platform/web_common.h"') |
| return content |
| |
| def _update_blink_build(self, content): |
| content = self._update_build(content) |
| |
| # Update visibility=[...] |
| content = content.replace('//third_party/WebKit/*', '//third_party/blink/*') |
| content = content.replace('//third_party/WebKit/Source/*', '//third_party/blink/renderer/*') |
| content = content.replace('//third_party/WebKit/public/*', '//third_party/blink/renderer/public/*') |
| |
| return self._update_basename(content) |
| |
| def _update_owners(self, content): |
| content = content.replace('//third_party/WebKit/Source', '//third_party/blink/renderer') |
| content = content.replace('//third_party/WebKit/common', '//third_party/blink/common') |
| content = content.replace('//third_party/WebKit/public', '//third_party/blink/renderer/public') |
| return content |
| |
| def _update_deps(self, content): |
| original_content = content |
| content = content.replace('third_party/WebKit/Source', 'third_party/blink/renderer') |
| content = content.replace('third_party/WebKit/common', 'third_party/blink/common') |
| content = content.replace('third_party/WebKit/public', 'third_party/blink/renderer/public') |
| content = content.replace('third_party/WebKit', 'third_party/blink') |
| if original_content == content: |
| return content |
| return self._update_basename(content) |
| |
| def _update_mojom(self, content): |
| content = content.replace('third_party/WebKit/public', 'third_party/blink/renderer/public') |
| return content |
| |
| def _update_typemap(self, content): |
| content = content.replace('//third_party/WebKit/Source', '//third_party/blink/renderer') |
| content = content.replace('//third_party/WebKit/common', '//third_party/blink/common') |
| content = content.replace('//third_party/WebKit/public', '//third_party/blink/renderer/public') |
| return self._update_basename(content) |
| |
| def _update_blink_build_py(self, content): |
| # We don't prepend 'third_party/blink/renderer/' to matched basenames |
| # because it won't affect build and manual update after the great mv is |
| # enough. |
| return self._update_basename(content) |
| |
| def _update_layout_tests(self, content): |
| return content.replace('file:///gen/third_party/WebKit/', 'file:///gen/third_party/blink/renderer/') |
| |
| def _update_basename(self, content): |
| return self._basename_re.sub(lambda match: self._basename_map[match.group(1)], content) |
| |
| @staticmethod |
| def _append_unless_upper_dir_exists(dirs, new_dir): |
| for i in range(0, len(dirs)): |
| if new_dir.startswith(dirs[i]): |
| return |
| if dirs[i].startswith(new_dir): |
| dirs[i] = new_dir |
| return |
| dirs.append(new_dir) |
| |
| def _update_file_content(self): |
| _log.info('Find *.gn, *.mojom, *.py, *.typemap, DEPS, and OWNERS ...') |
| files = self._fs.files_under( |
| self._repo_root, dirs_to_skip=['.git', 'out'], file_filter=self._filter_file) |
| _log.info('Scan contents of %d files ...', len(files)) |
| updated_deps_dirs = [] |
| for file_path in files: |
| file_type = FileType.detect(file_path) |
| original_content = self._fs.read_text_file(file_path) |
| content = original_content |
| if file_type == FileType.BUILD: |
| content = self._update_build(content) |
| elif file_type == FileType.BLINK_BUILD: |
| content = self._update_blink_build(content) |
| elif file_type == FileType.OWNERS: |
| content = self._update_owners(content) |
| elif file_type == FileType.DEPS: |
| if self._fs.dirname(file_path) == self._repo_root: |
| _log.info("Skip //DEPS") |
| continue |
| content = self._update_deps(content) |
| elif file_type == FileType.MOJOM: |
| content = self._update_mojom(content) |
| elif file_type == FileType.TYPEMAP: |
| content = self._update_typemap(content) |
| elif file_type == FileType.BLINK_BUILD_PY: |
| content = self._update_blink_build_py(content) |
| elif file_type == FileType.LAYOUT_TESTS_WITH_MOJOM: |
| content = self._update_layout_tests(content) |
| |
| if original_content == content: |
| continue |
| if self._options.run: |
| self._fs.write_text_file(file_path, content) |
| self._updated_files.append(file_path) |
| if file_type == FileType.DEPS: |
| self._append_unless_upper_dir_exists(updated_deps_dirs, self._fs.dirname(file_path)) |
| _log.info('Updated %s', self._shorten_path(file_path)) |
| return updated_deps_dirs |
| |
| def _update_cpp_includes_in_directories(self, dirs): |
| for dirname in dirs: |
| _log.info('Processing #include in %s ...', self._shorten_path(dirname)) |
| files = self._fs.files_under( |
| dirname, file_filter=lambda fs, _, basename: basename.endswith( |
| ('.h', '.cc', '.cpp', '.mm', '.cc.tmpl', '.cpp.tmpl', |
| '.h.tmpl', 'XPathGrammar.y', '.gperf'))) |
| for file_path in files: |
| original_content = self._fs.read_text_file(file_path) |
| |
| content = self._update_cpp_includes(original_content) |
| if file_path.endswith('.h') and '/third_party/WebKit/public/' in file_path.replace('\\', '/'): |
| content = self._update_basename_only_includes(content, file_path) |
| if file_path.endswith('.h') and '/third_party/WebKit/' in file_path.replace('\\', '/'): |
| content = self._update_include_guard(content, file_path) |
| |
| if original_content == content: |
| continue |
| if self._options.run: |
| self._fs.write_text_file(file_path, content) |
| self._updated_files.append(file_path) |
| _log.info('Updated %s', self._shorten_path(file_path)) |
| |
| def _replace_include_path(self, match): |
| include_or_import = match.group(1) |
| path = match.group(2) |
| |
| # If |path| starts with 'third_party/WebKit', we should adjust the |
| # directory name for third_party/blink, and replace its basename by |
| # self._basename_map. |
| # |
| # If |path| starts with a Blink-internal directory such as bindings, |
| # core, modules, platform, public, it refers to a checked-in file, or a |
| # generated file. For the former, we should add |
| # 'third_party/blink/renderer/' and replace the basename. For the |
| # latter, we should update the basename for a name mapped from an IDL |
| # renaming, and should not add 'third_party/blink/renderer'. |
| |
| if path.startswith('third_party/WebKit'): |
| path = path.replace('third_party/WebKit/Source', 'third_party/blink/renderer') |
| path = path.replace('third_party/WebKit/common', 'third_party/blink/common') |
| path = path.replace('third_party/WebKit/public', 'third_party/blink/renderer/public') |
| path = self._update_basename(path) |
| return '#%s "%s"' % (include_or_import, path) |
| |
| match = self._checked_in_header_re.search(path) |
| if match: |
| if match.group(1) in self._basename_map: |
| path = 'third_party/blink/renderer/' + path[:match.start(1)] + self._basename_map[match.group(1)] |
| else: |
| path = 'third_party/blink/renderer/' + path |
| elif 'core/inspector/protocol/' not in path: |
| basename_start = path.rfind('/') + 1 |
| basename = path[basename_start:] |
| if basename in self._idl_generated_impl_headers: |
| path = path[:basename_start] + self._basename_map[basename] |
| elif basename.startswith('V8'): |
| path = path[:basename_start] + NameStyleConverter(basename[:len(basename) - 2]).to_snake_case() + '.h' |
| return '#%s "%s"' % (include_or_import, path) |
| |
| def _update_cpp_includes(self, content): |
| pattern = re.compile(r'#(include|import)\s+"((bindings|core|modules|platform|public|' + |
| r'third_party/WebKit/(Source|common|public))/[-_\w/.]+)"') |
| return pattern.sub(self._replace_include_path, content) |
| |
| def _replace_basename_only_include(self, subdir, source_path, match): |
| source_basename = match.group(1) |
| if source_basename in self._basename_map: |
| return '#include "third_party/blink/renderer/public/%s/%s"' % (subdir, self._basename_map[source_basename]) |
| _log.warning('Basename-only %s in %s', match.group(0), self._shorten_path(source_path)) |
| return match.group(0) |
| |
| def _update_basename_only_includes(self, content, source_path): |
| if not source_path.endswith('.h') or '/third_party/WebKit/public/' not in source_path.replace('\\', '/'): |
| return |
| # In public/ header files, we should replace |#include "WebFoo.h"| |
| # with |#include "third_party/blink/renderer/public/platform-or-web/web_foo.h"| |
| subdir = self._fs.basename(self._fs.dirname(source_path)) |
| # subdir is 'web' or 'platform'. |
| return re.sub(r'#include\s+"(\w+\.h)"', |
| partial(self._replace_basename_only_include, subdir, source_path), content) |
| |
| def _update_include_guard(self, content, source_path): |
| current_guard = re.sub(r'[.]', '_', self._fs.basename(source_path)) |
| new_path = relative_dest(self._fs, self._fs.relpath( |
| source_path, start=self._fs.join(self._repo_root, 'third_party', 'WebKit'))) |
| new_guard = 'THIRD_PARTY_BLINK_' + re.sub(r'[\\/.]', '_', new_path.upper()) + '_' |
| content = re.sub(r'#ifndef\s+(WTF_)?' + current_guard, '#ifndef ' + new_guard, content); |
| content = re.sub(r'#define\s+(WTF_)?' + current_guard, '#define ' + new_guard, content); |
| content = re.sub(r'#endif\s+//\s+(WTF_)?' + current_guard, '#endif // ' + new_guard, content); |
| return content |
| |
| def _update_single_file_content(self, file_path, replace_list, should_write=True): |
| full_path = self._fs.join(self._repo_root, file_path) |
| original_content = self._fs.read_text_file(full_path) |
| content = original_content |
| for command in replace_list: |
| if isinstance(command, tuple): |
| src, dest = command |
| content = content.replace(src, dest) |
| elif callable(command): |
| content = command(content) |
| else: |
| raise TypeError('A tuple or a function is expected.') |
| if content != original_content: |
| if should_write: |
| self._fs.write_text_file(full_path, content) |
| self._updated_files.append(full_path) |
| _log.info('Updated %s', file_path) |
| else: |
| _log.warning('%s does not contain specified source strings.', file_path) |
| |
| |
| def main(): |
| logging.basicConfig(level=logging.DEBUG, |
| format='[%(asctime)s %(levelname)s %(name)s] %(message)s', |
| datefmt='%H:%M:%S') |
| parser = argparse.ArgumentParser(description='Blink source mover') |
| sub_parsers = parser.add_subparsers() |
| |
| update_parser = sub_parsers.add_parser('update') |
| update_parser.set_defaults(command='update') |
| update_parser.add_argument('--run', dest='run', action='store_true', |
| help='Update file contents') |
| |
| move_parser = sub_parsers.add_parser('move') |
| move_parser.set_defaults(command='move') |
| move_parser.add_argument('--git', dest='run_git', action='store_true', |
| help='Run |git mv| command instead of |mv|.') |
| |
| options = parser.parse_args() |
| mover = MoveBlinkSource(FileSystem(), options, get_chromium_src_dir()) |
| if options.command == 'update': |
| mover.update() |
| elif options.command == 'move': |
| mover.move() |
| |
| |
| if __name__ == '__main__': |
| main() |
