| #!/usr/bin/env python3 |
| # |
| # Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file |
| # for details. All rights reserved. Use of this source code is governed by a |
| # BSD-style license that can be found in the LICENSE file. |
| |
| import argparse |
| import io |
| import json |
| import os |
| import subprocess |
| import sys |
| import time |
| import utils |
| |
| import gn as gn_py |
| |
| HOST_OS = utils.GuessOS() |
| HOST_CPUS = utils.GuessCpus() |
| SCRIPT_DIR = os.path.dirname(sys.argv[0]) |
| DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..')) |
| AVAILABLE_ARCHS = utils.ARCH_FAMILY.keys() |
| |
| usage = """\ |
| usage: %%prog [options] [targets] |
| |
| This script invokes ninja to build Dart. |
| """ |
| |
| def BuildOptions(): |
| parser = argparse.ArgumentParser( |
| description='Runs GN (if necessary) followed by ninja', |
| formatter_class=argparse.ArgumentDefaultsHelpFormatter) |
| |
| config_group = parser.add_argument_group('Configuration Related Arguments') |
| gn_py.AddCommonConfigurationArgs(config_group) |
| |
| gn_group = parser.add_argument_group('GN Related Arguments') |
| gn_py.AddCommonGnOptionArgs(gn_group) |
| |
| other_group = parser.add_argument_group('Other Arguments') |
| gn_py.AddOtherArgs(other_group) |
| |
| other_group.add_argument("-j", |
| type=int, |
| help='Ninja -j option for Goma/RBE builds.', |
| default=200 if sys.platform == 'win32' else 1000) |
| other_group.add_argument("-l", |
| type=int, |
| help='Ninja -l option for Goma/RBE builds.', |
| default=64) |
| other_group.add_argument("--no-start-goma", |
| help="Don't try to start goma", |
| default=False, |
| dest='no_start_rbe', |
| action='store_true') |
| other_group.add_argument("--no-start-rbe", |
| help="Don't try to start rbe", |
| default=False, |
| action='store_true') |
| other_group.add_argument( |
| "--check-clean", |
| help="Check that a second invocation of Ninja has nothing to do", |
| default=False, |
| action='store_true') |
| |
| parser.add_argument('build_targets', nargs='*') |
| |
| return parser |
| |
| |
| def NotifyBuildDone(build_config, success, start): |
| if not success: |
| print("BUILD FAILED") |
| |
| sys.stdout.flush() |
| |
| # Display a notification if build time exceeded DART_BUILD_NOTIFICATION_DELAY. |
| notification_delay = float( |
| os.getenv('DART_BUILD_NOTIFICATION_DELAY', sys.float_info.max)) |
| if (time.time() - start) < notification_delay: |
| return |
| |
| if success: |
| message = 'Build succeeded.' |
| else: |
| message = 'Build failed.' |
| title = build_config |
| |
| command = None |
| if HOST_OS == 'macos': |
| # Use AppleScript to display a UI non-modal notification. |
| script = 'display notification "%s" with title "%s" sound name "Glass"' % ( |
| message, title) |
| command = "osascript -e '%s' &" % script |
| elif HOST_OS == 'linux': |
| if success: |
| icon = 'dialog-information' |
| else: |
| icon = 'dialog-error' |
| command = "notify-send -i '%s' '%s' '%s' &" % (icon, message, title) |
| elif HOST_OS == 'win32': |
| if success: |
| icon = 'info' |
| else: |
| icon = 'error' |
| command = ( |
| "powershell -command \"" |
| "[reflection.assembly]::loadwithpartialname('System.Windows.Forms')" |
| "| Out-Null;" |
| "[reflection.assembly]::loadwithpartialname('System.Drawing')" |
| "| Out-Null;" |
| "$n = new-object system.windows.forms.notifyicon;" |
| "$n.icon = [system.drawing.systemicons]::information;" |
| "$n.visible = $true;" |
| "$n.showballoontip(%d, '%s', '%s', " |
| "[system.windows.forms.tooltipicon]::%s);\"") % ( |
| 5000, # Notification stays on for this many milliseconds |
| message, |
| title, |
| icon) |
| |
| if command: |
| # Ignore return code, if this command fails, it doesn't matter. |
| os.system(command) |
| |
| def UseGoma(out_dir): |
| args_gn = os.path.join(out_dir, 'args.gn') |
| return 'use_goma = true' in open(args_gn, 'r').read() |
| |
| |
| def UseRBE(out_dir): |
| args_gn = os.path.join(out_dir, 'args.gn') |
| return 'use_rbe = true' in open(args_gn, 'r').read() |
| |
| |
| # Try to start RBE, but don't bail out if we can't. Instead print an error |
| # message, and let the build fail with its own error messages as well. |
| rbe_started = None |
| bootstrap_path = None |
| |
| |
| def StartRBE(out_dir, use_goma, env): |
| global rbe_started, bootstrap_path |
| rbe = 'goma' if use_goma else 'rbe' |
| if rbe_started: |
| if rbe_started != rbe: |
| print(f'Cannot mix RBE and Goma') |
| return False |
| return True |
| args_gn_path = os.path.join(out_dir, 'args.gn') |
| rbe_dir = None if use_goma else 'buildtools/reclient' |
| with open(args_gn_path, 'r') as fp: |
| for line in fp: |
| if ('goma_dir' if use_goma else 'rbe_dir') in line: |
| words = line.split() |
| rbe_dir = words[2][1:-1] # rbe_dir = "/path/to/rbe" |
| if not rbe_dir: |
| print(f'Could not find {rbe} for {out_dir}') |
| return False |
| if not os.path.exists(rbe_dir) or not os.path.isdir(rbe_dir): |
| print(f'Could not find {rbe} at {rbe_dir}') |
| return False |
| bootstrap = 'goma_ctl.py' if use_goma else 'bootstrap' |
| bootstrap_path = os.path.join(rbe_dir, bootstrap) |
| bootstrap_command = [bootstrap_path] |
| if use_goma: |
| bootstrap_command = ['python3', bootstrap_path, 'ensure_start'] |
| process = subprocess.Popen(bootstrap_command, env=env) |
| process.wait() |
| if process.returncode != 0: |
| print(f"Failed to start {rbe}") |
| return False |
| rbe_started = rbe |
| return True |
| |
| |
| def StopRBE(env): |
| global rbe_started, bootstrap_path |
| if rbe_started != 'rbe': |
| return |
| bootstrap_command = [bootstrap_path, '--shutdown'] |
| process = subprocess.Popen(bootstrap_command, env=env) |
| process.wait() |
| |
| |
| # Returns a tuple (build_config, command to run, whether rbe is used) |
| def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer, env): |
| build_config = utils.GetBuildConf(mode, arch, target_os, sanitizer) |
| out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer) |
| using_rbe = False |
| command = ['buildtools/ninja/ninja', '-C', out_dir] |
| if options.verbose: |
| command += ['-v'] |
| use_rbe = UseRBE(out_dir) |
| use_goma = UseGoma(out_dir) |
| if use_rbe or use_goma: |
| if options.no_start_rbe or StartRBE(out_dir, use_goma, env): |
| using_rbe = True |
| command += [('-j%s' % str(options.j))] |
| command += [('-l%s' % str(options.l))] |
| else: |
| exit(1) |
| command += targets |
| return (build_config, command, using_rbe) |
| |
| |
| def RunOneBuildCommand(build_config, args, env): |
| start_time = time.time() |
| print(' '.join(args)) |
| process = subprocess.Popen(args, env=env, stdin=None) |
| process.wait() |
| if process.returncode != 0: |
| NotifyBuildDone(build_config, success=False, start=start_time) |
| return 1 |
| else: |
| NotifyBuildDone(build_config, success=True, start=start_time) |
| |
| return 0 |
| |
| |
| def CheckCleanBuild(build_config, args, env): |
| args = args + ['-n', '-d', 'explain'] |
| print(' '.join(args)) |
| process = subprocess.Popen(args, |
| env=env, |
| stdout=subprocess.PIPE, |
| stderr=subprocess.PIPE, |
| stdin=None) |
| out, err = process.communicate() |
| process.wait() |
| if process.returncode != 0: |
| return 1 |
| if 'ninja: no work to do' not in out.decode('utf-8'): |
| print(err.decode('utf-8')) |
| return 1 |
| |
| return 0 |
| |
| |
| def SanitizerEnvironmentVariables(): |
| with io.open('tools/bots/test_matrix.json', encoding='utf-8') as fd: |
| config = json.loads(fd.read()) |
| env = dict() |
| for k, v in config['sanitizer_options'].items(): |
| env[str(k)] = str(v) |
| symbolizer_path = config['sanitizer_symbolizer'].get(HOST_OS, None) |
| if symbolizer_path: |
| symbolizer_path = str(os.path.join(DART_ROOT, symbolizer_path)) |
| env['ASAN_SYMBOLIZER_PATH'] = symbolizer_path |
| env['LSAN_SYMBOLIZER_PATH'] = symbolizer_path |
| env['MSAN_SYMBOLIZER_PATH'] = symbolizer_path |
| env['TSAN_SYMBOLIZER_PATH'] = symbolizer_path |
| env['UBSAN_SYMBOLIZER_PATH'] = symbolizer_path |
| return env |
| |
| |
| def Build(configs, env, options): |
| # Build regular configs. |
| rbe_builds = [] |
| for (build_config, args, rbe) in configs: |
| if args is None: |
| return 1 |
| if rbe: |
| rbe_builds.append([env, args]) |
| elif RunOneBuildCommand(build_config, args, env=env) != 0: |
| return 1 |
| |
| # Run RBE builds in parallel. |
| active_rbe_builds = [] |
| for (env, args) in rbe_builds: |
| print(' '.join(args)) |
| process = subprocess.Popen(args, env=env) |
| active_rbe_builds.append([args, process]) |
| while active_rbe_builds: |
| time.sleep(0.1) |
| for rbe_build in active_rbe_builds: |
| (args, process) = rbe_build |
| if process.poll() is not None: |
| print(' '.join(args) + " done.") |
| active_rbe_builds.remove(rbe_build) |
| if process.returncode != 0: |
| for (_, to_kill) in active_rbe_builds: |
| to_kill.terminate() |
| return 1 |
| |
| if options.check_clean: |
| for (build_config, args, rbe) in configs: |
| if CheckCleanBuild(build_config, args, env=env) != 0: |
| return 1 |
| |
| return 0 |
| |
| |
| def Main(): |
| starttime = time.time() |
| # Parse the options. |
| parser = BuildOptions() |
| options = parser.parse_args() |
| |
| targets = options.build_targets |
| |
| if not gn_py.ProcessOptions(options): |
| parser.print_help() |
| return 1 |
| |
| # If binaries are built with sanitizers we should use those flags. |
| # If the binaries are not built with sanitizers the flag should have no |
| # effect. |
| env = dict(os.environ) |
| env.update(SanitizerEnvironmentVariables()) |
| |
| # macOS's python sets CPATH, LIBRARY_PATH, SDKROOT implicitly. |
| # |
| # See: |
| # |
| # * https://openradar.appspot.com/radar?id=5608755232243712 |
| # * https://github.com/dart-lang/sdk/issues/52411 |
| # |
| # Remove these environment variables to avoid affecting clang's behaviors. |
| if sys.platform == 'darwin': |
| env.pop('CPATH', None) |
| env.pop('LIBRARY_PATH', None) |
| env.pop('SDKROOT', None) |
| |
| # Always run GN before building. |
| gn_py.RunGnOnConfiguredConfigurations(options, env) |
| |
| # Build all targets for each requested configuration. |
| configs = [] |
| for target_os in options.os: |
| for mode in options.mode: |
| for arch in options.arch: |
| for sanitizer in options.sanitizer: |
| configs.append( |
| BuildOneConfig(options, targets, target_os, mode, arch, |
| sanitizer, env)) |
| |
| exit_code = Build(configs, env, options) |
| |
| StopRBE(env) |
| |
| if exit_code == 0: |
| endtime = time.time() |
| print("The build took %.3f seconds" % (endtime - starttime)) |
| return exit_code |
| |
| |
| if __name__ == '__main__': |
| sys.exit(Main()) |
