| # Copyright (c) 2023, the Dart project authors. Please see the AUTHORS file |
| # for details. All rights reserved. Use of this source code is governed by a |
| # BSD-style license that can be found in the LICENSE file. |
| # |
| """Clang based C++ code indexer which produces xref.json.""" |
| from __future__ import annotations |
| |
| import glob |
| import json |
| import logging |
| import os |
| import platform |
| import re |
| import sys |
| import subprocess |
| |
| from contextlib import contextmanager |
| from dataclasses import dataclass, field |
| from typing import Dict, Optional, Set |
| |
| from marshmallow_dataclass import class_schema |
| from progress.bar import ShadyBar |
| |
| _libclang_path = None |
| _clang_include_dir = None |
| if platform.system() == 'Darwin': |
| _path_to_llvm = subprocess.check_output(['brew', '--prefix', 'llvm'], |
| encoding='utf-8').strip() |
| _clang_include_dir = glob.glob(f'{_path_to_llvm}/lib/clang/**/include')[0] |
| _libclang_path = f'{_path_to_llvm}/lib/libclang.dylib' |
| sys.path.append( |
| f'{_path_to_llvm}/lib/python{sys.version_info.major}.{sys.version_info.minor}/site-packages' |
| ) |
| |
| # pylint: disable=wrong-import-position,line-too-long |
| from clang.cindex import Config, CompilationDatabase, Cursor, CursorKind, Index, SourceLocation, TranslationUnit |
| |
| if _libclang_path is not None: |
| Config.set_library_file(_libclang_path) |
| |
| _DART_CONFIGURATION = 'Release' + ( |
| 'ARM64' if platform.uname().machine == 'arm64' else 'X64') |
| _DART_BUILD_DIR = ('xcodebuild' if platform.system() == 'Darwin' else |
| 'out') + '/' + _DART_CONFIGURATION |
| |
| |
| @contextmanager |
| def _change_working_directory_to(path): |
| oldpwd = os.getcwd() |
| os.chdir(path) |
| try: |
| yield |
| finally: |
| os.chdir(oldpwd) |
| |
| |
| def _create_compilation_database(): |
| """Create compilation database for the default configuration. |
| |
| Extacts compilation database (compile_commands.json) for the default |
| build configuration and filters it down to commands taht builds just |
| the core VM pieces in the host configuration. |
| """ |
| logging.info('Extracting compilation commands from build files for %s', |
| _DART_CONFIGURATION) |
| with _change_working_directory_to(_DART_BUILD_DIR): |
| commands = json.loads( |
| subprocess.check_output(['ninja', '-C', '.', '-t', 'compdb', |
| 'cxx'])) |
| pattern = re.compile( |
| r'libdart(_vm|_compiler)?_precompiler_host_targeting_host\.') |
| with open('compile_commands.json', 'w', encoding='utf-8') as outfile: |
| json.dump([ |
| cmd for cmd in commands |
| if pattern.search(cmd['command']) is not None |
| ], outfile) |
| |
| |
| def _get_current_commit_hash(): |
| return subprocess.check_output(['git', 'merge-base', 'main', 'HEAD'], |
| text=True).strip() |
| |
| |
| @dataclass |
| class _ClassInfo: |
| location: str |
| members: Dict[str, str] = field(default_factory=dict) |
| |
| |
| @dataclass |
| class Location: |
| """Symbol location referring to a line in a specific file.""" |
| filename: str |
| lineno: int |
| |
| |
| @dataclass |
| class SymbolsIndex: |
| """Index of C++ symbols extracted from source.""" |
| commit: str |
| files: list[str] = field(default_factory=list) |
| classes: Dict[str, _ClassInfo] = field(default_factory=dict) |
| functions: Dict[str, str] = field(default_factory=dict) |
| |
| def try_resolve(self, ref: str) -> Optional[Location]: |
| """Resolve the location of the given reference.""" |
| loc = self._try_resolve_impl(ref) |
| if loc is not None: |
| return self._location_from_string(loc) |
| return None |
| |
| def _try_resolve_impl(self, ref: str) -> Optional[str]: |
| if ref in self.functions: |
| return self.functions[ref] |
| if ref in self.classes: |
| return self.classes[ref].location |
| if '::' in ref: |
| (class_name, function_name) = ref.rsplit('::', 1) |
| if class_name in self.classes: |
| return self.classes[class_name].members.get(function_name) |
| return None |
| |
| def _location_from_string(self, loc: str) -> Location: |
| (file_idx, line_idx) = loc.split(':', 1) |
| return Location(self.files[int(file_idx)], int(line_idx)) |
| |
| |
| class _Indexer: |
| symbols_index: SymbolsIndex |
| |
| processed_files: Set[str] |
| classes_by_usr: Dict[str, _ClassInfo] |
| |
| name_stack: list[str] |
| info_stack: list[_ClassInfo] |
| files_seen_in_unit: Set[str] |
| |
| def __init__(self): |
| self.symbols_index = SymbolsIndex(commit=_get_current_commit_hash()) |
| |
| self.processed_files = set() |
| self.classes_by_usr = {} |
| self.files_index = {} |
| |
| self.name_stack = [] |
| self.info_stack = [] |
| self.files_seen_in_unit = set() |
| |
| def index(self, unit: TranslationUnit): |
| """Index the given translation unit and append new symbols to index.""" |
| self.name_stack.clear() |
| self.info_stack.clear() |
| self.files_seen_in_unit.clear() |
| self._recurse(unit.cursor) |
| self.processed_files |= self.files_seen_in_unit |
| |
| def _recurse(self, cursor: Cursor): |
| name = "" |
| kind = cursor.kind |
| |
| if cursor.location.file is not None: |
| name = cursor.location.file.name |
| if name in self.processed_files or not name.startswith('../..'): |
| return |
| self.files_seen_in_unit.add(name) |
| |
| if kind == CursorKind.CLASS_DECL: |
| if not cursor.is_definition(): |
| return |
| usr = cursor.get_usr() |
| if usr in self.classes_by_usr: |
| return |
| self.name_stack.append(cursor.spelling) |
| class_name = '::'.join(self.name_stack) |
| class_info = _ClassInfo(self._format_location(cursor.location)) |
| self.info_stack.append(class_info) |
| self.symbols_index.classes[class_name] = class_info |
| self.classes_by_usr[usr] = class_info |
| elif kind == CursorKind.NAMESPACE: |
| self.name_stack.append(cursor.spelling) |
| elif kind == CursorKind.FUNCTION_DECL and cursor.is_definition(): |
| namespace_prefix = "" |
| if cursor.semantic_parent.kind == CursorKind.NAMESPACE: |
| namespace_prefix = '::'.join(self.name_stack) + ( |
| '::' if len(self.name_stack) > 0 else "") |
| function_name = namespace_prefix + cursor.spelling |
| self.symbols_index.functions[function_name] = self._format_location( |
| cursor.location) |
| return |
| elif kind == CursorKind.CXX_METHOD and cursor.is_definition(): |
| parent = cursor.semantic_parent |
| if parent.kind == CursorKind.CLASS_DECL: |
| class_info_or_none = self.classes_by_usr.get(parent.get_usr()) |
| if class_info_or_none is None: |
| return |
| class_info_or_none.members[ |
| cursor.spelling] = self._format_location(cursor.location) |
| return |
| elif kind == CursorKind.VAR_DECL and cursor.is_definition(): |
| parent = cursor.semantic_parent |
| if parent.kind == CursorKind.CLASS_DECL: |
| class_info_or_none = self.classes_by_usr.get(parent.get_usr()) |
| if class_info_or_none is None: |
| return |
| class_info_or_none.members[ |
| cursor.spelling] = self._format_location(cursor.location) |
| |
| for child in cursor.get_children(): |
| self._recurse(child) |
| |
| if kind == CursorKind.NAMESPACE: |
| self.name_stack.pop() |
| elif kind == CursorKind.CLASS_DECL: |
| self.name_stack.pop() |
| self.info_stack.pop() |
| |
| def _format_location(self, loc: SourceLocation): |
| file_name = loc.file.name |
| lineno = loc.line |
| return f'{self._get_file_index(file_name)}:{lineno}' |
| |
| def _get_file_index(self, file_name: str): |
| index = self.files_index.get(file_name) |
| if index is None: |
| index = len(self.symbols_index.files) |
| self.files_index[file_name] = index |
| self.symbols_index.files.append( |
| os.path.relpath(os.path.abspath(file_name), |
| os.path.abspath('../..'))) |
| return index |
| |
| |
| def _index_source() -> SymbolsIndex: |
| indexer = _Indexer() |
| |
| _create_compilation_database() |
| with _change_working_directory_to(_DART_BUILD_DIR): |
| index = Index.create() |
| compdb = CompilationDatabase.fromDirectory('.') |
| |
| commands = list(compdb.getAllCompileCommands()) |
| with ShadyBar('Indexing', |
| max=len(commands), |
| suffix='%(percent)d%% eta %(eta_td)s') as progress_bar: |
| for command in commands: |
| args = [ |
| arg for arg in command.arguments |
| if arg.startswith('-I') or arg.startswith('-W') or |
| arg.startswith('-D') or arg.startswith('-i') or |
| arg.startswith('sdk/') or arg.startswith('-std') |
| ] + [ |
| '-Wno-macro-redefined', '-Wno-unused-const-variable', |
| '-Wno-unused-function', '-Wno-unused-variable' |
| ] |
| |
| if _clang_include_dir is not None: |
| args.append(f'-I{_clang_include_dir}') |
| |
| unit = index.parse(command.filename, args=args) |
| for diag in unit.diagnostics: |
| print(diag.format()) |
| |
| indexer.index(unit) |
| progress_bar.next() |
| |
| return indexer.symbols_index |
| |
| |
| _SymbolsIndexSchema = class_schema(SymbolsIndex)() |
| |
| |
| def load_index(filename: str) -> SymbolsIndex: |
| """Load symbols index from the given file. |
| |
| If index is out of date or missing it will be generated. |
| """ |
| index: SymbolsIndex |
| |
| if os.path.exists(filename): |
| with open(filename, 'r', encoding='utf-8') as json_file: |
| index = _SymbolsIndexSchema.loads(json_file.read()) |
| if _get_current_commit_hash() == index.commit: |
| logging.info('Loaded symbols index from %s', filename) |
| return index |
| logging.warning( |
| '%s is generated for commit %s while current commit is %s', |
| filename, index.commit, _get_current_commit_hash()) |
| |
| index = _index_source() |
| with open(filename, 'w', encoding='utf-8') as json_file: |
| json_file.write(_SymbolsIndexSchema.dumps(index)) |
| logging.info( |
| 'Successfully indexed C++ source and written symbols index into %s', |
| filename) |
| return index |
