| # Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file | |
| # for details. All rights reserved. Use of this source code is governed by a | |
| # BSD-style license that can be found in the LICENSE file. | |
| # This file contains a set of utilities functions used by other Python-based | |
| # scripts. | |
| from __future__ import print_function | |
| import contextlib | |
| import datetime | |
| from functools import total_ordering | |
| import glob | |
| import importlib.util | |
| import importlib.machinery | |
| import json | |
| import os | |
| import platform | |
| import re | |
| import shutil | |
| import subprocess | |
| import sys | |
| import tarfile | |
| import tempfile | |
| import uuid | |
| try: | |
| # Not available on Windows. | |
| import resource | |
| except: | |
| pass | |
| SEMANTIC_VERSION_PATTERN = r'^(?P<major>0|[1-9]\d*)\.(?P<minor>0|[1-9]\d*)\.(?P<patch>0|[1-9]\d*)(?:-(?P<prerelease>(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*)(?:\.(?:0|[1-9]\d*|\d*[a-zA-Z-][0-9a-zA-Z-]*))*))?(?:\+(?P<buildmetadata>[0-9a-zA-Z-]+(?:\.[0-9a-zA-Z-]+)*))?$' | |
| # To eliminate clashing with older archived builds on bleeding edge we add | |
| # a base number bigger the largest svn revision (this also gives us an easy | |
| # way of seeing if an archive comes from git based or svn based commits). | |
| GIT_NUMBER_BASE = 100000 | |
| # Mapping table between build mode and build configuration. | |
| BUILD_MODES = { | |
| 'debug': 'Debug', | |
| 'release': 'Release', | |
| 'product': 'Product', | |
| } | |
| # Mapping table between build mode and build configuration. | |
| BUILD_SANITIZERS = { | |
| None: '', | |
| 'none': '', | |
| 'asan': 'ASAN', | |
| 'hwasan': 'HWASAN', | |
| 'lsan': 'LSAN', | |
| 'msan': 'MSAN', | |
| 'tsan': 'TSAN', | |
| 'ubsan': 'UBSAN', | |
| } | |
| # Mapping table between OS and build output location. | |
| BUILD_ROOT = { | |
| 'win32': 'out', | |
| 'linux': 'out', | |
| 'freebsd': 'out', | |
| 'macos': 'xcodebuild', | |
| } | |
| # Note: gn expects these to be lower case. | |
| ARCH_FAMILY = { | |
| 'ia32': 'ia32', | |
| 'x64': 'ia32', | |
| 'arm': 'arm', | |
| 'arm64': 'arm', | |
| 'arm64e': 'arm', | |
| 'arm_x64': 'arm', | |
| 'arm_arm64': 'arm', | |
| 'simarm': 'ia32', | |
| 'simarm64': 'ia32', | |
| 'simarm_x64': 'ia32', | |
| 'simarm_arm64': 'arm', | |
| 'x64c': 'ia32', | |
| 'arm64c': 'arm', | |
| 'simarm64c': 'ia32', | |
| 'simriscv32': 'ia32', | |
| 'simriscv64': 'ia32', | |
| 'simx64': 'arm', | |
| 'simx64c': 'arm', | |
| 'riscv32': 'riscv', | |
| 'riscv64': 'riscv', | |
| } | |
| BASE_DIR = os.path.abspath(os.path.join(os.curdir, '..')) | |
| DART_DIR = os.path.abspath(os.path.join(__file__, '..', '..')) | |
| VERSION_FILE = os.path.join(DART_DIR, 'tools', 'VERSION') | |
| def GetArchFamily(arch): | |
| return ARCH_FAMILY[arch] | |
| def GetBuildDir(host_os): | |
| return BUILD_ROOT[host_os] | |
| def GetBuildMode(mode): | |
| return BUILD_MODES[mode] | |
| def GetBuildSanitizer(sanitizer): | |
| return BUILD_SANITIZERS[sanitizer] | |
| def GetBaseDir(): | |
| return BASE_DIR | |
| def load_source(modname, filename): | |
| loader = importlib.machinery.SourceFileLoader(modname, filename) | |
| spec = importlib.util.spec_from_file_location(modname, | |
| filename, | |
| loader=loader) | |
| module = importlib.util.module_from_spec(spec) | |
| # The module is always executed and not cached in sys.modules. | |
| # Uncomment the following line to cache the module. | |
| # sys.modules[module.__name__] = module | |
| loader.exec_module(module) | |
| return module | |
| def GetBotUtils(repo_path=DART_DIR): | |
| '''Dynamically load the tools/bots/bot_utils.py python module.''' | |
| return load_source('bot_utils', | |
| os.path.join(repo_path, 'tools', 'bots', 'bot_utils.py')) | |
| def GetMinidumpUtils(repo_path=DART_DIR): | |
| '''Dynamically load the tools/minidump.py python module.''' | |
| return load_source('minidump', | |
| os.path.join(repo_path, 'tools', 'minidump.py')) | |
| @total_ordering | |
| class Version(object): | |
| def __init__(self, | |
| channel=None, | |
| major=None, | |
| minor=None, | |
| patch=None, | |
| prerelease=None, | |
| prerelease_patch=None, | |
| version=None): | |
| self.channel = channel | |
| self.major = major | |
| self.minor = minor | |
| self.patch = patch | |
| self.prerelease = prerelease | |
| self.prerelease_patch = prerelease_patch | |
| if version: | |
| self.set_version(version) | |
| def set_version(self, version): | |
| match = re.match(SEMANTIC_VERSION_PATTERN, version) | |
| assert match, '%s must be a valid version' % version | |
| self.channel = 'stable' | |
| self.major = match['major'] | |
| self.minor = match['minor'] | |
| self.patch = match['patch'] | |
| self.prerelease = '0' | |
| self.prerelease_patch = '0' | |
| if match['prerelease']: | |
| subversions = match['prerelease'].split('.') | |
| self.prerelease = subversions[0] | |
| self.prerelease_patch = subversions[1] | |
| self.channel = subversions[2] | |
| def __str__(self): | |
| result = '%s.%s.%s' % (self.major, self.minor, self.patch) | |
| if self.channel != 'stable': | |
| result += '-%s.%s.%s' % (self.prerelease, self.prerelease_patch, | |
| self.channel) | |
| return result | |
| def __eq__(self, other): | |
| return self.channel == other.channel and \ | |
| self.major == other.major and \ | |
| self.minor == other.minor and \ | |
| self.patch == other.patch and \ | |
| self.prerelease == other.prerelease and \ | |
| self.prerelease_patch == other.prerelease_patch | |
| def __lt__(self, other): | |
| if int(self.major) < int(other.major): | |
| return True | |
| if int(self.major) > int(other.major): | |
| return False | |
| if int(self.minor) < int(other.minor): | |
| return True | |
| if int(self.minor) > int(other.minor): | |
| return False | |
| if int(self.patch) < int(other.patch): | |
| return True | |
| if int(self.patch) > int(other.patch): | |
| return False | |
| # The stable channel is ahead of the other channels on the same triplet. | |
| if self.channel != 'stable' and other.channel == 'stable': | |
| return True | |
| if self.channel == 'stable' and other.channel != 'stable': | |
| return False | |
| # The main channel is ahead of the other channels on the same triplet. | |
| if self.channel != 'main' and other.channel == 'main': | |
| return True | |
| if self.channel == 'main' and other.channel != 'main': | |
| return False | |
| # The be channel existed before it was renamed to main. | |
| if self.channel != 'be' and other.channel == 'be': | |
| return True | |
| if self.channel == 'be' and other.channel != 'be': | |
| return False | |
| if int(self.prerelease) < int(other.prerelease): | |
| return True | |
| if int(self.prerelease) > int(other.prerelease): | |
| return False | |
| if int(self.prerelease_patch) < int(other.prerelease_patch): | |
| return True | |
| if int(self.prerelease_patch) > int(other.prerelease_patch): | |
| return False | |
| return False | |
| # Try to guess the host operating system. | |
| def GuessOS(): | |
| os_id = platform.system() | |
| if os_id == 'Linux': | |
| return 'linux' | |
| elif os_id == 'Darwin': | |
| return 'macos' | |
| elif os_id == 'Windows' or os_id == 'Microsoft': | |
| # On Windows Vista platform.system() can return 'Microsoft' with some | |
| # versions of Python, see http://bugs.python.org/issue1082 for details. | |
| return 'win32' | |
| elif os_id == 'FreeBSD': | |
| return 'freebsd' | |
| elif os_id == 'OpenBSD': | |
| return 'openbsd' | |
| elif os_id == 'SunOS': | |
| return 'solaris' | |
| return None | |
| # Runs true if the currently executing python interpreter is running under | |
| # Rosetta. I.e., python3 is an x64 executable and we're on an arm64 Mac. | |
| def IsRosetta(): | |
| if platform.system() == 'Darwin': | |
| p = subprocess.Popen(['sysctl', '-in', 'sysctl.proc_translated'], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.STDOUT) | |
| output, _ = p.communicate() | |
| return output.decode('utf-8').strip() == '1' | |
| return False | |
| # Returns the architectures that can run on the current machine. | |
| def HostArchitectures(): | |
| m = platform.machine() | |
| if platform.system() == 'Darwin': | |
| if m == 'arm64' or IsRosetta(): | |
| # ARM64 Macs also support X64. | |
| # macOS 26 supports arm64e. Before this, we need to treat arm64e as a cross build. | |
| major = int(platform.mac_ver()[0].split('.')[0]) | |
| if major >= 26: | |
| return ['arm64', 'arm64e', 'x64'] | |
| else: | |
| return ['arm64', 'x64'] | |
| if m == 'x86_64': | |
| # X64 Macs no longer support IA32. | |
| return ['x64'] | |
| # Icky use of CIPD_ARCHITECTURE should be effectively dead whenever the | |
| # Python on bots becomes native ARM64. | |
| if ((platform.system() == 'Windows') and | |
| (os.environ.get("CIPD_ARCHITECTURE") == "arm64")): | |
| # ARM64 Windows also can emulate X64. | |
| return ['arm64', 'x64'] | |
| if m in ['aarch64', 'arm64', 'arm64e', 'ARM64']: | |
| return ['arm64'] | |
| if m in ['armv7l', 'armv8l']: | |
| return ['arm'] | |
| if m in ['i386', 'i686', 'ia32', 'x86']: | |
| return ['x86', 'ia32'] | |
| if m in ['x64', 'x86-64', 'x86_64', 'amd64', 'AMD64']: | |
| return ['x64', 'x86', 'ia32'] | |
| if m in ['riscv64']: | |
| return ['riscv64'] | |
| raise Exception('Failed to determine host architectures for %s %s', | |
| platform.machine(), platform.system()) | |
| # Try to guess the host architecture. | |
| def GuessArchitecture(): | |
| return HostArchitectures()[0] | |
| # Try to guess the number of cpus on this machine. | |
| def GuessCpus(): | |
| if os.getenv('DART_NUMBER_OF_CORES') is not None: | |
| return int(os.getenv('DART_NUMBER_OF_CORES')) | |
| if os.path.exists('/proc/cpuinfo'): | |
| return int( | |
| subprocess.check_output( | |
| 'grep -E \'^processor\' /proc/cpuinfo | wc -l', shell=True)) | |
| if os.path.exists('/usr/bin/hostinfo'): | |
| return int( | |
| subprocess.check_output( | |
| '/usr/bin/hostinfo |' | |
| ' grep "processors are logically available." |' | |
| ' awk "{ print \\$1 }"', | |
| shell=True)) | |
| win_cpu_count = os.getenv("NUMBER_OF_PROCESSORS") | |
| if win_cpu_count: | |
| return int(win_cpu_count) | |
| return 2 | |
| # Returns true if we're running under Windows. | |
| def IsWindows(): | |
| return GuessOS() == 'win32' | |
| def IsCrossBuild(target_os, arch): | |
| if (target_os not in [None, 'host']) and (target_os != GuessOS()): | |
| return True | |
| if arch.startswith('sim'): | |
| return False | |
| if arch.endswith('c'): | |
| # Strip 'compressed' suffix. | |
| arch = arch[:-1] | |
| if arch in HostArchitectures(): | |
| return False | |
| return True | |
| def GetBuildConf(mode, arch, conf_os=None, sanitizer=None): | |
| if conf_os is not None and conf_os != GuessOS() and conf_os != 'host': | |
| os_fragment = conf_os.title() | |
| if (conf_os == 'ios_simulator'): | |
| os_fragment = 'IosSim' | |
| if (conf_os == 'watchos_simulator'): | |
| os_fragment = 'WatchosSim' | |
| return '{}{}{}{}'.format(GetBuildMode(mode), os_fragment, | |
| GetBuildSanitizer(sanitizer), arch.upper()) | |
| # Ask for a cross build if the host and target architectures don't match. | |
| cross_build = '' | |
| if IsCrossBuild(conf_os, arch): | |
| cross_build = 'X' | |
| return '{}{}{}{}'.format(GetBuildMode(mode), GetBuildSanitizer(sanitizer), | |
| cross_build, arch.upper()) | |
| def GetBuildRoot(host_os, mode=None, arch=None, target_os=None, sanitizer=None): | |
| build_root = GetBuildDir(host_os) | |
| if mode: | |
| build_root = os.path.join( | |
| build_root, GetBuildConf(mode, arch, target_os, sanitizer)) | |
| return build_root | |
| def GetVersion(no_git_hash=False, version_file=None): | |
| version = ReadVersionFile(version_file) | |
| if not version: | |
| return None | |
| suffix = '' | |
| if version.channel in ['main', 'be']: | |
| suffix = '-edge' if no_git_hash else '-edge.{}'.format(GetGitRevision()) | |
| elif version.channel in ('beta', 'dev'): | |
| suffix = '-{}.{}.{}'.format(version.prerelease, | |
| version.prerelease_patch, version.channel) | |
| else: | |
| assert version.channel == 'stable' | |
| return '{}.{}.{}{}'.format(version.major, version.minor, version.patch, | |
| suffix) | |
| def GetChannel(version_file=None): | |
| version = ReadVersionFile(version_file) | |
| return version.channel | |
| def ReadVersionFile(version_file=None): | |
| def match_against(pattern, file_content): | |
| match = re.search(pattern, file_content, flags=re.MULTILINE) | |
| if match: | |
| return match.group(1) | |
| return None | |
| if version_file == None: | |
| version_file = VERSION_FILE | |
| content = None | |
| try: | |
| with open(version_file) as fd: | |
| content = fd.read() | |
| except: | |
| print('Warning: Could not read VERSION file ({})'.format(version_file)) | |
| return None | |
| channel = match_against('^CHANNEL ([A-Za-z0-9]+)$', content) | |
| major = match_against('^MAJOR (\\d+)$', content) | |
| minor = match_against('^MINOR (\\d+)$', content) | |
| patch = match_against('^PATCH (\\d+)$', content) | |
| prerelease = match_against('^PRERELEASE (\\d+)$', content) | |
| prerelease_patch = match_against('^PRERELEASE_PATCH (\\d+)$', content) | |
| if (channel and major and minor and prerelease and prerelease_patch): | |
| return Version(channel, major, minor, patch, prerelease, | |
| prerelease_patch) | |
| print('Warning: VERSION file ({}) has wrong format'.format(version_file)) | |
| return None | |
| def GetGitRevision(repo_path=DART_DIR): | |
| p = subprocess.Popen(['git', 'rev-parse', 'HEAD'], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| shell=IsWindows(), | |
| cwd=repo_path) | |
| out, err = p.communicate() | |
| # TODO(https://github.com/dart-lang/sdk/issues/51865): Don't ignore errors. | |
| # if p.wait() != 0: | |
| # raise Exception('git rev-parse failed: ' + str(err)) | |
| revision = out.decode('utf-8').strip() | |
| # We expect a full git hash | |
| if len(revision) != 40: | |
| print('Warning: Could not parse git commit, output was {}'.format( | |
| revision), | |
| file=sys.stderr) | |
| return None | |
| return revision | |
| def GetShortGitHash(repo_path=DART_DIR): | |
| p = subprocess.Popen(['git', 'rev-parse', '--short=10', 'HEAD'], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| shell=IsWindows(), | |
| cwd=repo_path) | |
| out, err = p.communicate() | |
| if p.wait() != 0: | |
| # TODO(https://github.com/dart-lang/sdk/issues/51865): Don't ignore errors. | |
| # raise Exception('git rev-parse failed: ' + str(err)) | |
| return None | |
| revision = out.decode('utf-8').strip() | |
| return revision | |
| def GetGitTimestamp(repo_path=DART_DIR): | |
| p = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:%cd'], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| shell=IsWindows(), | |
| cwd=repo_path) | |
| out, err = p.communicate() | |
| if p.wait() != 0: | |
| # TODO(https://github.com/dart-lang/sdk/issues/51865): Don't ignore errors. | |
| # raise Exception('git log failed: ' + str(err)) | |
| return None | |
| timestamp = out.decode('utf-8').strip() | |
| return timestamp | |
| def GetGitTimestampForDpkg(repo_path=DART_DIR): | |
| p = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:%cD'], | |
| stdout=subprocess.PIPE, | |
| stderr=subprocess.PIPE, | |
| shell=IsWindows(), | |
| cwd=repo_path) | |
| out, err = p.communicate() | |
| if p.wait() != 0: | |
| # TODO(https://github.com/dart-lang/sdk/issues/51865): Don't ignore errors. | |
| # raise Exception('git log failed: ' + str(err)) | |
| return None | |
| timestamp = out.decode('utf-8').strip() | |
| return timestamp | |
| # TODO(42528): Can we remove this? It's basically just an alias for Exception. | |
| class Error(Exception): | |
| pass | |
| def IsCrashExitCode(exit_code): | |
| if IsWindows(): | |
| return 0x80000000 & exit_code | |
| return exit_code < 0 | |
| def DiagnoseExitCode(exit_code, command): | |
| if IsCrashExitCode(exit_code): | |
| sys.stderr.write( | |
| 'Command: {}\nCRASHED with exit code {} (0x{:x})\n'.format( | |
| ' '.join(command), exit_code, exit_code & 0xffffffff)) | |
| def CheckedInSdkPath(): | |
| tools_dir = os.path.dirname(os.path.realpath(__file__)) | |
| return os.path.join(tools_dir, 'sdks', 'dart-sdk') | |
| def CheckedInSdkExecutable(): | |
| name = 'dart' | |
| if IsWindows(): | |
| name = 'dart.exe' | |
| return os.path.join(CheckedInSdkPath(), 'bin', name) | |
| def CheckLinuxCoreDumpPattern(fatal=False): | |
| core_pattern_file = '/proc/sys/kernel/core_pattern' | |
| core_pattern = open(core_pattern_file).read() | |
| expected_core_pattern = 'core.%p' | |
| if core_pattern.strip() != expected_core_pattern: | |
| message = ( | |
| 'Invalid core_pattern configuration. ' | |
| 'The configuration of core dump handling is *not* correct for ' | |
| 'a buildbot. The content of {0} must be "{1}" instead of "{2}".'. | |
| format(core_pattern_file, expected_core_pattern, core_pattern)) | |
| if fatal: | |
| raise Exception(message) | |
| print(message) | |
| return False | |
| return True | |
| class TempDir(object): | |
| def __init__(self, prefix=''): | |
| self._temp_dir = None | |
| self._prefix = prefix | |
| def __enter__(self): | |
| self._temp_dir = tempfile.mkdtemp(self._prefix) | |
| return self._temp_dir | |
| def __exit__(self, *_): | |
| shutil.rmtree(self._temp_dir, ignore_errors=True) | |
| class UnexpectedCrash(object): | |
| def __init__(self, test, pid, *binaries): | |
| self.test = test | |
| self.pid = pid | |
| self.binaries = binaries | |
| def __str__(self): | |
| return 'Crash({}: {} {})'.format(self.test, self.pid, | |
| ', '.join(self.binaries)) | |
| class PosixCoreDumpEnabler(object): | |
| def __init__(self): | |
| self._old_limits = None | |
| def __enter__(self): | |
| self._old_limits = resource.getrlimit(resource.RLIMIT_CORE) | |
| resource.setrlimit(resource.RLIMIT_CORE, (-1, -1)) | |
| def __exit__(self, *_): | |
| if self._old_limits != None: | |
| resource.setrlimit(resource.RLIMIT_CORE, self._old_limits) | |
| class LinuxCoreDumpEnabler(PosixCoreDumpEnabler): | |
| def __enter__(self): | |
| # Bump core limits to unlimited if core_pattern is correctly configured. | |
| if CheckLinuxCoreDumpPattern(fatal=False): | |
| super(LinuxCoreDumpEnabler, self).__enter__() | |
| def __exit__(self, *args): | |
| CheckLinuxCoreDumpPattern(fatal=False) | |
| super(LinuxCoreDumpEnabler, self).__exit__(*args) | |
| class WindowsCoreDumpEnabler(object): | |
| """This enabler assumes that Dart binary was built with Crashpad support. | |
| In this case DART_CRASHPAD_CRASHES_DIR environment variable allows to | |
| specify the location of Crashpad crashes database. Actual minidumps will | |
| be written into reports subfolder of the database. | |
| """ | |
| CRASHPAD_DB_FOLDER = os.path.join(DART_DIR, 'crashes') | |
| DUMPS_FOLDER = os.path.join(CRASHPAD_DB_FOLDER, 'reports') | |
| def __init__(self): | |
| pass | |
| def __enter__(self): | |
| print('INFO: Enabling coredump archiving into {}'.format( | |
| WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER)) | |
| os.environ[ | |
| 'DART_CRASHPAD_CRASHES_DIR'] = WindowsCoreDumpEnabler.CRASHPAD_DB_FOLDER | |
| def __exit__(self, *_): | |
| del os.environ['DART_CRASHPAD_CRASHES_DIR'] | |
| def TryUnlink(file): | |
| try: | |
| os.unlink(file) | |
| except Exception as error: | |
| print('ERROR: Failed to remove {}: {}'.format(file, error)) | |
| class BaseCoreDumpArchiver(object): | |
| """This class reads coredumps file written by UnexpectedCrashDumpArchiver | |
| into the current working directory and uploads all cores and binaries | |
| listed in it into Cloud Storage (see | |
| pkg/test_runner/lib/src/test_progress.dart). | |
| """ | |
| # test.dart will write a line for each unexpected crash into this file. | |
| _UNEXPECTED_CRASHES_FILE = 'unexpected-crashes' | |
| def __init__(self, search_dir, output_directory): | |
| self._bucket = 'dart-temp-crash-archive' | |
| self._binaries_dir = os.getcwd() | |
| self._search_dir = search_dir | |
| self._output_directory = output_directory | |
| def _safe_cleanup(self): | |
| try: | |
| return self._cleanup() | |
| except Exception as error: | |
| print('ERROR: Failure during cleanup: {}'.format(error)) | |
| return False | |
| def __enter__(self): | |
| print('INFO: Core dump archiving is activated') | |
| # Cleanup any stale files | |
| if self._safe_cleanup(): | |
| print('WARNING: Found and removed stale coredumps') | |
| def __exit__(self, *_): | |
| try: | |
| crashes = self._find_unexpected_crashes() | |
| if crashes: | |
| # If we get a ton of crashes, only archive 10 dumps. | |
| archive_crashes = crashes[:10] | |
| print('Archiving coredumps for crash (if possible):') | |
| for crash in archive_crashes: | |
| print('----> {}'.format(crash)) | |
| sys.stdout.flush() | |
| self._archive(archive_crashes) | |
| else: | |
| print('INFO: No unexpected crashes recorded') | |
| dumps = self._find_all_coredumps() | |
| if dumps: | |
| print('INFO: However there are {} core dumps found'.format( | |
| len(dumps))) | |
| for dump in dumps: | |
| print('INFO: -> {}'.format(dump)) | |
| print() | |
| except Exception as error: | |
| print('ERROR: Failed to archive crashes: {}'.format(error)) | |
| raise | |
| finally: | |
| self._safe_cleanup() | |
| def _archive(self, crashes): | |
| files = set() | |
| missing = [] | |
| for crash in crashes: | |
| files.update(crash.binaries) | |
| core = self._find_coredump_file(crash) | |
| if core: | |
| files.add(core) | |
| else: | |
| missing.append(crash) | |
| if self._output_directory is not None and self._is_shard(): | |
| print( | |
| "INFO: Moving collected dumps and binaries into output directory\n" | |
| "INFO: They will be uploaded to isolate server. Look for \"isolated" | |
| " out\" under the failed step on the build page.\n" | |
| "INFO: For more information see runtime/docs/infra/coredumps.md" | |
| ) | |
| self._move(files) | |
| else: | |
| print( | |
| "INFO: Uploading collected dumps and binaries into Cloud Storage\n" | |
| "INFO: Use `gcloud storage cp from-url to-path` to download them.\n" | |
| "INFO: For more information see runtime/docs/infra/coredumps.md" | |
| ) | |
| self._upload(files) | |
| if missing: | |
| self._report_missing_crashes(missing, throw=False) | |
| # todo(athom): move the logic to decide where to copy core dumps into the recipes. | |
| def _is_shard(self): | |
| return 'BUILDBOT_BUILDERNAME' not in os.environ | |
| def _report_missing_crashes(self, missing, throw=False): | |
| missing_as_string = ', '.join([str(c) for c in missing]) | |
| other_files = list(glob.glob(os.path.join(self._search_dir, '*'))) | |
| sys.stderr.write( | |
| "Could not find crash dumps for '{}' in search directory '{}'.\n" | |
| "Existing files which *did not* match the pattern inside the search " | |
| "directory are are:\n {}\n".format(missing_as_string, | |
| self._search_dir, | |
| '\n '.join(other_files))) | |
| # TODO: Figure out why windows coredump generation does not work. | |
| # See http://dartbug.com/36469 | |
| if throw and GuessOS() != 'win32': | |
| raise Exception( | |
| 'Missing crash dumps for: {}'.format(missing_as_string)) | |
| def _get_file_name(self, file): | |
| # Sanitize the name: actual cores follow 'core.%d' pattern, crashed | |
| # binaries are copied next to cores and named | |
| # 'binary.<mode>_<arch>_<binary_name>'. | |
| # This should match the code in testing/dart/test_progress.dart | |
| name = os.path.basename(file) | |
| (prefix, suffix) = name.split('.', 1) | |
| is_binary = prefix == 'binary' | |
| if is_binary: | |
| (mode, arch, binary_name) = suffix.split('_', 2) | |
| name = binary_name | |
| return (name, is_binary) | |
| def _move(self, files): | |
| for file in files: | |
| print('+++ Moving {} to output_directory ({})'.format( | |
| file, self._output_directory)) | |
| (name, is_binary) = self._get_file_name(file) | |
| destination = os.path.join(self._output_directory, name) | |
| shutil.move(file, destination) | |
| if is_binary and os.path.exists(file + '.pdb'): | |
| # Also move a PDB file if there is one. | |
| pdb = os.path.join(self._output_directory, name + '.pdb') | |
| shutil.move(file + '.pdb', pdb) | |
| def _tar(self, file): | |
| (name, is_binary) = self._get_file_name(file) | |
| tarname = '{}.tar.gz'.format(name) | |
| # Compress the file. | |
| tar = tarfile.open(tarname, mode='w:gz') | |
| tar.add(file, arcname=name) | |
| if is_binary and os.path.exists(file + '.pdb'): | |
| # Also add a PDB file if there is one. | |
| tar.add(file + '.pdb', arcname=name + '.pdb') | |
| tar.close() | |
| return tarname | |
| def _upload(self, files): | |
| bot_utils = GetBotUtils() | |
| gsutil = bot_utils.GSUtil() | |
| storage_path = '{}/{}/'.format(self._bucket, uuid.uuid4()) | |
| gs_prefix = 'gs://{}'.format(storage_path) | |
| http_prefix = 'https://storage.cloud.google.com/{}'.format(storage_path) | |
| print('\n--- Uploading into {} ({}) ---'.format(gs_prefix, http_prefix)) | |
| for file in files: | |
| tarname = self._tar(file) | |
| # Remove / from absolute path to not have // in gs path. | |
| gs_url = '{}{}'.format(gs_prefix, tarname) | |
| http_url = '{}{}'.format(http_prefix, tarname) | |
| try: | |
| gsutil.upload(tarname, gs_url) | |
| print('+++ Uploaded {} ({})'.format(gs_url, http_url)) | |
| except Exception as error: | |
| print('!!! Failed to upload {}, error: {}'.format( | |
| tarname, error)) | |
| TryUnlink(tarname) | |
| print('--- Done ---\n') | |
| def _find_all_coredumps(self): | |
| """Return coredumps that were recorded (if supported by the platform). | |
| This method will be overridden by concrete platform specific implementations. | |
| """ | |
| return [] | |
| def _find_unexpected_crashes(self): | |
| """Load coredumps file. Each line has the following format: | |
| test-name,pid,binary-file1,binary-file2,... | |
| """ | |
| try: | |
| with open(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE) as f: | |
| return [ | |
| UnexpectedCrash(*ln.strip('\n').split(',')) | |
| for ln in f.readlines() | |
| ] | |
| except: | |
| return [] | |
| def _cleanup(self): | |
| found = False | |
| if os.path.exists(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE): | |
| os.unlink(BaseCoreDumpArchiver._UNEXPECTED_CRASHES_FILE) | |
| found = True | |
| for binary in glob.glob(os.path.join(self._binaries_dir, 'binary.*')): | |
| found = True | |
| TryUnlink(binary) | |
| return found | |
| class PosixCoreDumpArchiver(BaseCoreDumpArchiver): | |
| def __init__(self, search_dir, output_directory): | |
| super(PosixCoreDumpArchiver, self).__init__(search_dir, | |
| output_directory) | |
| def _cleanup(self): | |
| found = super(PosixCoreDumpArchiver, self)._cleanup() | |
| for core in glob.glob(os.path.join(self._search_dir, 'core.*')): | |
| found = True | |
| TryUnlink(core) | |
| return found | |
| def _find_coredump_file(self, crash): | |
| core_filename = os.path.join(self._search_dir, | |
| 'core.{}'.format(crash.pid)) | |
| if os.path.exists(core_filename): | |
| return core_filename | |
| class LinuxCoreDumpArchiver(PosixCoreDumpArchiver): | |
| def __init__(self, output_directory): | |
| super(LinuxCoreDumpArchiver, self).__init__(os.getcwd(), | |
| output_directory) | |
| class MacOSCoreDumpArchiver(PosixCoreDumpArchiver): | |
| def __init__(self, output_directory): | |
| super(MacOSCoreDumpArchiver, self).__init__('/cores', output_directory) | |
| class WindowsCoreDumpArchiver(BaseCoreDumpArchiver): | |
| def __init__(self, output_directory): | |
| super(WindowsCoreDumpArchiver, self).__init__( | |
| WindowsCoreDumpEnabler.DUMPS_FOLDER, output_directory) | |
| self._dumps_by_pid = None | |
| # Find CDB.exe in the win_toolchain that we are using. | |
| def _find_cdb(self): | |
| win_toolchain_json_path = os.path.join(DART_DIR, 'build', | |
| 'win_toolchain.json') | |
| if not os.path.exists(win_toolchain_json_path): | |
| return None | |
| with open(win_toolchain_json_path, 'r') as f: | |
| win_toolchain_info = json.loads(f.read()) | |
| win_sdk_path = win_toolchain_info['win_sdk'] | |
| # We assume that we are running on 64-bit Windows. | |
| # Note: x64 CDB can work with both X64 and IA32 dumps. | |
| cdb_path = os.path.join(win_sdk_path, 'Debuggers', 'x64', 'cdb.exe') | |
| if not os.path.exists(cdb_path): | |
| return None | |
| return cdb_path | |
| CDBG_PROMPT_RE = re.compile(r'^\d+:\d+>') | |
| def _dump_all_stacks(self): | |
| # On Windows due to crashpad integration crashes do not produce any | |
| # stacktraces. Dump stack traces from dumps Crashpad collected using | |
| # CDB (if available). | |
| cdb_path = self._find_cdb() | |
| if cdb_path is None: | |
| return | |
| dumps = self._find_all_coredumps() | |
| if not dumps: | |
| return | |
| print('### Collected {} crash dumps'.format(len(dumps))) | |
| for dump in dumps: | |
| print() | |
| print('### Dumping stacks from {} using CDB'.format(dump)) | |
| cdb_output = subprocess.check_output( | |
| '"{}" -z "{}" -kqm -c "!uniqstack -b -v -p;qd"'.format( | |
| cdb_path, dump), | |
| stderr=subprocess.STDOUT) | |
| # Extract output of uniqstack from the whole output of CDB. | |
| output = False | |
| for line in cdb_output.split('\n'): | |
| if re.match(WindowsCoreDumpArchiver.CDBG_PROMPT_RE, line): | |
| output = True | |
| elif line.startswith('quit:'): | |
| break | |
| elif output: | |
| print(line) | |
| print() | |
| print('#############################################') | |
| print() | |
| def __exit__(self, *args): | |
| try: | |
| self._dump_all_stacks() | |
| except Exception as error: | |
| print('ERROR: Unable to dump stacks from dumps: {}'.format(error)) | |
| super(WindowsCoreDumpArchiver, self).__exit__(*args) | |
| def _cleanup(self): | |
| found = super(WindowsCoreDumpArchiver, self)._cleanup() | |
| for core in glob.glob(os.path.join(self._search_dir, '*')): | |
| found = True | |
| TryUnlink(core) | |
| return found | |
| def _find_all_coredumps(self): | |
| pattern = os.path.join(self._search_dir, '*.dmp') | |
| return [core_filename for core_filename in glob.glob(pattern)] | |
| def _find_coredump_file(self, crash): | |
| if self._dumps_by_pid is None: | |
| # If this function is invoked the first time then look through the directory | |
| # that contains crashes for all dump files and collect pid -> filename | |
| # mapping. | |
| self._dumps_by_pid = {} | |
| minidump = GetMinidumpUtils() | |
| pattern = os.path.join(self._search_dir, '*.dmp') | |
| for core_filename in glob.glob(pattern): | |
| pid = minidump.GetProcessIdFromDump(core_filename) | |
| if pid != -1: | |
| self._dumps_by_pid[str(pid)] = core_filename | |
| if crash.pid in self._dumps_by_pid: | |
| return self._dumps_by_pid[crash.pid] | |
| def _report_missing_crashes(self, missing, throw=False): | |
| # Let's only print the debugging information and not throw. We'll do more | |
| # validation for werfault.exe and throw afterwards. | |
| super(WindowsCoreDumpArchiver, self)._report_missing_crashes( | |
| missing, throw=False) | |
| if throw: | |
| missing_as_string = ', '.join([str(c) for c in missing]) | |
| raise Exception( | |
| 'Missing crash dumps for: {}'.format(missing_as_string)) | |
| class IncreasedNumberOfFileDescriptors(object): | |
| def __init__(self, nofiles): | |
| self._old_limits = None | |
| self._limits = (nofiles, nofiles) | |
| def __enter__(self): | |
| self._old_limits = resource.getrlimit(resource.RLIMIT_NOFILE) | |
| resource.setrlimit(resource.RLIMIT_NOFILE, self._limits) | |
| def __exit__(self, *_): | |
| resource.setrlimit(resource.RLIMIT_CORE, self._old_limits) | |
| @contextlib.contextmanager | |
| def NooptContextManager(): | |
| yield | |
| def CoreDumpArchiver(args): | |
| enabled = '--copy-coredumps' in args | |
| prefix = '--output-directory=' | |
| output_directory = next( | |
| (arg[len(prefix):] for arg in args if arg.startswith(prefix)), None) | |
| if not enabled: | |
| return (NooptContextManager(),) | |
| osname = GuessOS() | |
| if osname == 'linux': | |
| return (LinuxCoreDumpEnabler(), LinuxCoreDumpArchiver(output_directory)) | |
| elif osname == 'macos': | |
| return (PosixCoreDumpEnabler(), MacOSCoreDumpArchiver(output_directory)) | |
| elif osname == 'win32': | |
| return (WindowsCoreDumpEnabler(), | |
| WindowsCoreDumpArchiver(output_directory)) | |
| # We don't have support for MacOS yet. | |
| return (NooptContextManager(),) | |
| def FileDescriptorLimitIncreaser(): | |
| osname = GuessOS() | |
| if osname == 'macos': | |
| return IncreasedNumberOfFileDescriptors(nofiles=10000) | |
| assert osname in ('linux', 'win32') | |
| # We don't have support for MacOS yet. | |
| return NooptContextManager() | |
| def Main(): | |
| print('GuessOS() -> ', GuessOS()) | |
| print('GuessArchitecture() -> ', GuessArchitecture()) | |
| print('GuessCpus() -> ', GuessCpus()) | |
| print('IsWindows() -> ', IsWindows()) | |
| print('GetGitRevision() -> ', GetGitRevision()) | |
| print('GetGitTimestamp() -> ', GetGitTimestamp()) | |
| print('ReadVersionFile() -> ', ReadVersionFile()) | |
| if __name__ == '__main__': | |
| Main() |