blob: 895c698c52a2cd19b2c2dce2b21a63aa51d79fed [file] [log] [blame] [edit]
#!/usr/bin/env python
#
# Copyright (c) 2017, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import io
import json
import multiprocessing
import optparse
import os
import subprocess
import sys
import time
import utils
HOST_OS = utils.GuessOS()
HOST_CPUS = utils.GuessCpus()
SCRIPT_DIR = os.path.dirname(sys.argv[0])
DART_ROOT = os.path.realpath(os.path.join(SCRIPT_DIR, '..'))
AVAILABLE_ARCHS = utils.ARCH_FAMILY.keys()
usage = """\
usage: %%prog [options] [targets]
This script invokes ninja to build Dart.
"""
def BuildOptions():
result = optparse.OptionParser(usage=usage)
result.add_option(
"-a",
"--arch",
help='Target architectures (comma-separated).',
metavar='[all,' + ','.join(AVAILABLE_ARCHS) + ']',
default=utils.GuessArchitecture())
result.add_option(
"-b",
"--bytecode",
help='Build with the kernel bytecode interpreter. DEPRECATED.',
default=False,
action='store_true')
result.add_option(
"-j", type=int, help='Ninja -j option for Goma builds.', default=1000)
result.add_option(
"-l", type=int, help='Ninja -l option for Goma builds.', default=64)
result.add_option(
"-m",
"--mode",
help='Build variants (comma-separated).',
metavar='[all,debug,release,product]',
default='debug')
result.add_option(
"--no-start-goma",
help="Don't try to start goma",
default=False,
action='store_true')
result.add_option(
"--os",
help='Target OSs (comma-separated).',
metavar='[all,host,android]',
default='host')
result.add_option(
"--sanitizer",
type=str,
help='Build variants (comma-separated).',
metavar='[all,none,asan,lsan,msan,tsan,ubsan]',
default='none')
# TODO(38701): Remove this option once the NNBD SDK is stable/performant
# and there is no need to build a legacy version of the SDK for comparison
# purposes.
result.add_option(
"--no-nnbd",
help='Build the Legacy (pre NNBD) version of the SDK.',
default=False,
action='store_true')
result.add_option(
"-v",
"--verbose",
help='Verbose output.',
default=False,
action="store_true")
return result
def ProcessOsOption(os_name):
if os_name == 'host':
return HOST_OS
return os_name
def ProcessOptions(options, args):
if options.arch == 'all':
options.arch = 'ia32,x64,simarm,simarm64'
if options.mode == 'all':
options.mode = 'debug,release,product'
if options.os == 'all':
options.os = 'host,android'
if options.sanitizer == 'all':
options.sanitizer = 'none,asan,lsan,msan,tsan,ubsan'
options.mode = options.mode.split(',')
options.arch = options.arch.split(',')
options.os = options.os.split(',')
options.sanitizer = options.sanitizer.split(',')
for mode in options.mode:
if not mode in ['debug', 'release', 'product']:
print("Unknown mode %s" % mode)
return False
for i, arch in enumerate(options.arch):
if not arch in AVAILABLE_ARCHS:
# Normalise to lower case form to make it less case-picky.
arch_lower = arch.lower()
if arch_lower in AVAILABLE_ARCHS:
options.arch[i] = arch_lower
continue
print("Unknown arch %s" % arch)
return False
options.os = [ProcessOsOption(os_name) for os_name in options.os]
for os_name in options.os:
if not os_name in ['android', 'freebsd', 'linux', 'macos', 'win32']:
print("Unknown os %s" % os_name)
return False
if os_name != HOST_OS:
if os_name != 'android':
print("Unsupported target os %s" % os_name)
return False
if not HOST_OS in ['linux', 'macos']:
print("Cross-compilation to %s is not supported on host os %s."
% (os_name, HOST_OS))
return False
if not arch in [
'ia32', 'x64', 'arm', 'arm_x64', 'armv6', 'arm64'
]:
print(
"Cross-compilation to %s is not supported for architecture %s."
% (os_name, arch))
return False
# We have not yet tweaked the v8 dart build to work with the Android
# NDK/SDK, so don't try to build it.
if not args:
print(
"For android builds you must specify a target, such as 'runtime'."
)
return False
return True
def NotifyBuildDone(build_config, success, start):
if not success:
print("BUILD FAILED")
sys.stdout.flush()
# Display a notification if build time exceeded DART_BUILD_NOTIFICATION_DELAY.
notification_delay = float(
os.getenv('DART_BUILD_NOTIFICATION_DELAY', sys.float_info.max))
if (time.time() - start) < notification_delay:
return
if success:
message = 'Build succeeded.'
else:
message = 'Build failed.'
title = build_config
command = None
if HOST_OS == 'macos':
# Use AppleScript to display a UI non-modal notification.
script = 'display notification "%s" with title "%s" sound name "Glass"' % (
message, title)
command = "osascript -e '%s' &" % script
elif HOST_OS == 'linux':
if success:
icon = 'dialog-information'
else:
icon = 'dialog-error'
command = "notify-send -i '%s' '%s' '%s' &" % (icon, message, title)
elif HOST_OS == 'win32':
if success:
icon = 'info'
else:
icon = 'error'
command = (
"powershell -command \""
"[reflection.assembly]::loadwithpartialname('System.Windows.Forms')"
"| Out-Null;"
"[reflection.assembly]::loadwithpartialname('System.Drawing')"
"| Out-Null;"
"$n = new-object system.windows.forms.notifyicon;"
"$n.icon = [system.drawing.systemicons]::information;"
"$n.visible = $true;"
"$n.showballoontip(%d, '%s', '%s', "
"[system.windows.forms.tooltipicon]::%s);\"") % (
5000, # Notification stays on for this many milliseconds
message,
title,
icon)
if command:
# Ignore return code, if this command fails, it doesn't matter.
os.system(command)
def GenerateBuildfilesIfNeeded():
if os.path.exists(utils.GetBuildDir(HOST_OS)):
return True
command = [
'python',
os.path.join(DART_ROOT, 'tools', 'generate_buildfiles.py')
]
print("Running " + ' '.join(command))
process = subprocess.Popen(command)
process.wait()
if process.returncode != 0:
print("Tried to generate missing buildfiles, but failed. "
"Try running manually:\n\t$ " + ' '.join(command))
return False
return True
def RunGNIfNeeded(out_dir, target_os, mode, arch, dont_use_nnbd, sanitizer):
if os.path.isfile(os.path.join(out_dir, 'args.gn')):
return
gn_os = 'host' if target_os == HOST_OS else target_os
gn_command = [
'python',
os.path.join(DART_ROOT, 'tools', 'gn.py'),
'--sanitizer',
sanitizer,
'-m',
mode,
'-a',
arch,
'--os',
gn_os,
'-v',
]
if dont_use_nnbd:
gn_command.append('--no-nnbd')
process = subprocess.Popen(gn_command)
process.wait()
if process.returncode != 0:
print("Tried to run GN, but it failed. Try running it manually: \n\t$ "
+ ' '.join(gn_command))
def UseGoma(out_dir):
args_gn = os.path.join(out_dir, 'args.gn')
return 'use_goma = true' in open(args_gn, 'r').read()
# Try to start goma, but don't bail out if we can't. Instead print an error
# message, and let the build fail with its own error messages as well.
goma_started = False
def EnsureGomaStarted(out_dir):
global goma_started
if goma_started:
return True
args_gn_path = os.path.join(out_dir, 'args.gn')
goma_dir = None
with open(args_gn_path, 'r') as fp:
for line in fp:
if 'goma_dir' in line:
words = line.split()
goma_dir = words[2][1:-1] # goma_dir = "/path/to/goma"
if not goma_dir:
print('Could not find goma for ' + out_dir)
return False
if not os.path.exists(goma_dir) or not os.path.isdir(goma_dir):
print('Could not find goma at ' + goma_dir)
return False
goma_ctl = os.path.join(goma_dir, 'goma_ctl.py')
goma_ctl_command = [
'python',
goma_ctl,
'ensure_start',
]
process = subprocess.Popen(goma_ctl_command)
process.wait()
if process.returncode != 0:
print(
"Tried to run goma_ctl.py, but it failed. Try running it manually: "
+ "\n\t" + ' '.join(goma_ctl_command))
return False
goma_started = True
return True
# Returns a tuple (build_config, command to run, whether goma is used)
def BuildOneConfig(options, targets, target_os, mode, arch, sanitizer):
build_config = utils.GetBuildConf(mode, arch, target_os, sanitizer,
options.no_nnbd)
out_dir = utils.GetBuildRoot(HOST_OS, mode, arch, target_os, sanitizer,
options.no_nnbd)
using_goma = False
# TODO(zra): Remove auto-run of gn, replace with prompt for user to run
# gn.py manually.
RunGNIfNeeded(out_dir, target_os, mode, arch, options.no_nnbd, sanitizer)
command = ['ninja', '-C', out_dir]
if options.verbose:
command += ['-v']
if UseGoma(out_dir):
if options.no_start_goma or EnsureGomaStarted(out_dir):
using_goma = True
command += [('-j%s' % str(options.j))]
command += [('-l%s' % str(options.l))]
else:
# If we couldn't ensure that goma is started, let the build start, but
# slowly so we can see any helpful error messages that pop out.
command += ['-j1']
command += targets
return (build_config, command, using_goma)
def RunOneBuildCommand(build_config, args, env):
start_time = time.time()
print(' '.join(args))
process = subprocess.Popen(args, env=env, stdin=None)
process.wait()
if process.returncode != 0:
NotifyBuildDone(build_config, success=False, start=start_time)
return 1
else:
NotifyBuildDone(build_config, success=True, start=start_time)
return 0
def RunOneGomaBuildCommand(options):
(env, args) = options
try:
print(' '.join(args))
process = subprocess.Popen(args, env=env, stdin=None)
process.wait()
print(' '.join(args) + " done.")
return process.returncode
except KeyboardInterrupt:
return 1
def SanitizerEnvironmentVariables():
with io.open('tools/bots/test_matrix.json', encoding='utf-8') as fd:
config = json.loads(fd.read())
env = dict()
for k, v in config['sanitizer_options'].items():
env[str(k)] = str(v)
symbolizer_path = config['sanitizer_symbolizer'].get(HOST_OS, None)
if symbolizer_path:
symbolizer_path = str(os.path.join(DART_ROOT, symbolizer_path))
env['ASAN_SYMBOLIZER_PATH'] = symbolizer_path
env['LSAN_SYMBOLIZER_PATH'] = symbolizer_path
env['MSAN_SYMBOLIZER_PATH'] = symbolizer_path
env['TSAN_SYMBOLIZER_PATH'] = symbolizer_path
env['UBSAN_SYMBOLIZER_PATH'] = symbolizer_path
return env
def Main():
starttime = time.time()
# Parse the options.
parser = BuildOptions()
(options, args) = parser.parse_args()
if not ProcessOptions(options, args):
parser.print_help()
return 1
# Determine which targets to build. By default we build the "all" target.
if len(args) == 0:
targets = ['all']
else:
targets = args
if not GenerateBuildfilesIfNeeded():
return 1
# If binaries are built with sanitizers we should use those flags.
# If the binaries are not built with sanitizers the flag should have no
# effect.
env = dict(os.environ)
env.update(SanitizerEnvironmentVariables())
# Build all targets for each requested configuration.
configs = []
for target_os in options.os:
for mode in options.mode:
for arch in options.arch:
for sanitizer in options.sanitizer:
configs.append(
BuildOneConfig(options, targets, target_os, mode, arch,
sanitizer))
# Build regular configs.
goma_builds = []
for (build_config, args, goma) in configs:
if args is None:
return 1
if goma:
goma_builds.append([env, args])
elif RunOneBuildCommand(build_config, args, env=env) != 0:
return 1
# Run goma builds in parallel.
pool = multiprocessing.Pool(multiprocessing.cpu_count())
results = pool.map(RunOneGomaBuildCommand, goma_builds, chunksize=1)
for r in results:
if r != 0:
return 1
endtime = time.time()
print("The build took %.3f seconds" % (endtime - starttime))
return 0
if __name__ == '__main__':
sys.exit(Main())