blob: 3a1c85285f717cdb34e217cdd928f0527e64f486 [file] [log] [blame]
# Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import imp
import os
import re
import shutil
import shlex
import subprocess
import sys
import tempfile
import zipfile
import config_parser
# We expect the tools directory from the dart repo to be checked out into:
# ../../tools
DART_DIR = os.path.abspath(
os.path.normpath(os.path.join(__file__, '..', '..', '..')))
UTILS_PATH = os.path.join(DART_DIR, 'tools', '')
BOT_UTILS_PATH = os.path.join(DART_DIR, 'tools', 'bots', '')
if os.path.isfile(UTILS_PATH):
utils = imp.load_source('utils', UTILS_PATH)
print 'error: %s not found' % UTILS_PATH
if os.path.isfile(BOT_UTILS_PATH):
bot_utils = imp.load_source('bot_utils', BOT_UTILS_PATH)
print 'error: %s not found' % BOT_UTILS_PATH
# We are deliberately not using bot utils from the dart repo.
PACKAGES_BUILDER = r'packages-(windows|linux|mac)(-repo)?(-sample)?-(.*)'
'dart-protobuf' : 'protobuf',
'polymer-dart' : 'polymer',
'serialization.dart' : 'serialization',
'unittest-stable' : 'unittest'
# Some packages need all tests run sequentially, due to side effects.
# This list only affects tests run with 'pub run test'.
class BotInfo(object):
Stores the info extracted from the bot name
- system: windows, linux, mac
- package-name
def __init__(self, system, package_name, is_repo, is_sample):
self.system = system
self.package_name = NAME_OVERRIDES.get(package_name,
package_name.replace('-', '_'))
self.is_repo = is_repo
self.is_sample = is_sample
def __str__(self):
return "System: %s, Package-name: %s, Repo: %s, Sample: %s" % (
self.system, self.package_name, self.is_repo, self.is_sample)
def GetBotInfo():
name = os.environ.get('BUILDBOT_BUILDERNAME')
if not name:
print ("BUILDBOT_BUILDERNAME not defined. "
"Expected pattern of the form: %s" % PACKAGES_BUILDER)
builder_pattern = re.match(PACKAGES_BUILDER, name)
if builder_pattern:
is_repo = is not None
is_sample = is not None
return BotInfo(,,
class BuildStep(object):
A context manager for handling build steps.
When the context manager is entered, it prints the "@@@BUILD_STEP __@@@"
message. If it exits from an error being raised it displays the
"@@@STEP_FAILURE@@@" message.
If swallow_error is True, then this will catch and discard any OSError that
is thrown. This lets you run later BuildSteps if the current one fails.
def __init__(self, name, swallow_error=False): = name
self.swallow_error = swallow_error
def __enter__(self):
print '@@@BUILD_STEP %s@@@' %
def __exit__(self, type, value, traceback):
if value:
print '@@@STEP_FAILURE@@@'
if self.swallow_error and isinstance(value, OSError):
return True
class TempDir(object):
def __init__(self, prefix=''):
self._temp_dir = None
self._prefix = prefix
def __enter__(self):
self._temp_dir = tempfile.mkdtemp(self._prefix)
return self._temp_dir
def __exit__(self, *_):
shutil.rmtree(self._temp_dir, ignore_errors=True)
class ChangedWorkingDirectory(object):
def __init__(self, working_directory):
self._working_directory = working_directory
def __enter__(self):
self._old_cwd = os.getcwd()
print "Enter directory = ", self._working_directory
def __exit__(self, *_):
print "Enter directory = ", self._old_cwd
def RunProcess(command, shell=False, extra_env=None):
Runs command.
If a non-zero exit code is returned, raises an OSError with errno as the exit
env = dict(os.environ)
env['TERM'] = 'nocolor'
if 'GIT_USER_AGENT' in env:
del env['GIT_USER_AGENT']
if extra_env:
print "Running: %s" % ' '.join(command)
print "env: %s" % str(env)
exit_code =, env=env, shell=shell)
if exit_code != 0:
raise OSError(exit_code)
def GetSDK(bot_info):
with BuildStep('Get sdk'):
namer = bot_utils.GCSNamer(channel=bot_utils.Channel.DEV)
# TODO(ricow): Be smarter here, only download if new.
build_root = GetBuildRoot(bot_info)
SafeDelete(os.path.join(build_root, 'dart-sdk'), bot_info)
if not os.path.exists(build_root):
local_zip = os.path.join(build_root, '')
gsutils = bot_utils.GSUtil()
namer.sdk_zipfilepath('latest', bot_info.system,
'ia32', 'release'),
if bot_info.system == 'windows':
with zipfile.ZipFile(local_zip, 'r') as zip_file:
# We don't keep the execution bit if we use python's zipfile on posix.
RunProcess(['unzip', local_zip, '-d', build_root])
pub = GetPub(bot_info)
RunProcess([pub, '--version'])
def GetPackagePath(bot_info):
if bot_info.is_repo:
return os.path.join('pkg', bot_info.package_name)
return os.path.join('third_party', 'pkg', bot_info.package_name)
def GetBuildRoot(bot_info):
system = bot_info.system
if system == 'windows':
system = 'win32'
if system == 'mac':
system = 'macos'
return utils.GetBuildRoot(system, mode='release', arch='ia32',
def SafeDelete(path, bot_info):
if bot_info.system == 'windows':
if os.path.exists(path):
args = ['cmd.exe', '/c', 'rmdir', '/q', '/s', path]
shutil.rmtree(path, ignore_errors=True)
def GetPackageCopy(bot_info):
build_root = GetBuildRoot(bot_info)
package_copy = os.path.join(build_root, 'package_copy')
package_path = GetPackagePath(bot_info)
copy_path = os.path.join(package_copy, bot_info.package_name)
SafeDelete(package_copy, bot_info)
no_git = shutil.ignore_patterns('.git')
shutil.copytree(package_path, copy_path, symlinks=False, ignore=no_git)
return copy_path
def GetSdkBin(bot_info):
return os.path.join(os.getcwd(), GetBuildRoot(bot_info),
'dart-sdk', 'bin')
def GetPub(bot_info):
executable = 'pub.bat' if bot_info.system == 'windows' else 'pub'
return os.path.join(GetSdkBin(bot_info), executable)
def GetPubEnv(bot_info):
pub_cache = os.path.join(os.getcwd(), GetBuildRoot(bot_info), 'pub_cache')
return {'PUB_CACHE': pub_cache, 'PUB_ENVIRONMENT': 'dart_bots'}
# _RunPubCacheRepair and _CheckPubCacheCorruption are not used right now, but we
# keep them around because they provide an easy way to diagnose and fix issues
# in the bots.
def _RunPubCacheRepair(bot_info, path):
pub = GetPub(bot_info)
extra_env = GetPubEnv(bot_info)
with BuildStep('Pub cache repair'):
# For now, assume pub
with ChangedWorkingDirectory(path):
args = [pub, 'cache', 'repair']
RunProcess(args, extra_env=extra_env)
corruption_checks = 0
def _CheckPubCacheCorruption(bot_info, path):
extra_env = GetPubEnv(bot_info)
global corruption_checks
corruption_checks += 1
with BuildStep('Check pub cache corruption %d' % corruption_checks):
with ChangedWorkingDirectory(path):
packages = os.path.join(
extra_env['PUB_CACHE'], 'hosted', '')
print '\nLooking for packages in %s:' % str(packages)
if not os.path.exists(packages):
print "cache directory doesn't exist"
for package in os.listdir(packages):
if 'unittest-' in package:
exists = os.path.exists(
os.path.join(packages, package, 'lib', 'unittest.dart'))
print '- ok: ' if exists else '- bad: ',
print os.path.join(package, 'lib', 'unittest.dart')
print ''
def RunPubUpgrade(bot_info, path):
pub = GetPub(bot_info)
extra_env = GetPubEnv(bot_info)
with BuildStep('Pub upgrade'):
# For now, assume pub
with ChangedWorkingDirectory(path):
args = [pub, 'upgrade', '--no-packages-dir']
RunProcess(args, extra_env=extra_env)
def RunPubBuild(bot_info, path, folder, mode=None):
skip_pub_build = ['dart-protobuf', 'rpc']
with BuildStep('Pub build on %s' % folder):
if bot_info.package_name in skip_pub_build:
print "Not running pub build"
pub = GetPub(bot_info)
extra_env = GetPubEnv(bot_info)
with ChangedWorkingDirectory(path):
# run pub-build on the web folder
if os.path.exists(folder):
args = [pub, 'build']
if mode:
args.append('--mode=%s' % mode)
if folder != 'web':
RunProcess(args, extra_env=extra_env)
# Major hack
def FixupTestControllerJS(package_path):
if os.path.exists(os.path.join(package_path, 'packages', 'unittest')):
test_controller = os.path.join(package_path, 'packages', 'unittest',
dart_controller = os.path.join('tools', 'testing', 'dart',
print 'Hack test controller by copying of %s to %s' % (dart_controller,
shutil.copy(dart_controller, test_controller)
print "No unittest to patch, do you even have tests"
'windows': ['ff', 'chrome', 'ie10'],
'linux': ['d8', 'ff', 'chrome'],
'mac': ['safari'],
is_first_test_run = True
def LogsArgument():
global is_first_test_run
if is_first_test_run:
is_first_test_run = False
return []
return ['--append_logs']
def RunPackageTesting(bot_info, package_path, folder='test'):
package_name = os.path.basename(package_path)
if package_name == '':
# when package_path had a trailing slash
package_name = os.path.basename(os.path.dirname(package_path))
if folder == 'build/test':
suffix = ' under build'
package_root = os.path.join(package_path, folder, 'packages')
package_arg = '--package-root=%s' % package_root
suffix = ''
package_spec_file = os.path.join(package_path, '.packages')
package_arg = '--packages=%s' % package_spec_file
# Note: we use package_name/package_name/folder and not package_name/folder on
# purpose. The first package_name denotes the suite, the second is part of the
# path we want to match. Without the second package_name, we may match tests
# that contain "folder" further down. So if folder is "test",
# "package_name/test" matches "package_name/build/test", but
# "package_name/package_name/test" does not.
standard_args = ['--arch=ia32',
'--suite-dir=%s' % package_path,
'--use-sdk', '--report', '--progress=buildbot',
'--write-debug-log', '-v',
'%s/%s/%s/' % (package_name, package_name, folder)]
with BuildStep('Test vm release mode%s' % suffix, swallow_error=True):
args = [sys.executable, 'tools/',
'-mrelease', '-rvm', '-cnone'] + standard_args
# For easy integration testing we give access to the sdk bin directory.
# This only makes sense on vm testing.
extra_env = { 'DART_SDK_BIN' : GetSdkBin(bot_info) }
RunProcess(args, extra_env=extra_env)
with BuildStep('Test analyzer%s' % suffix, swallow_error=True):
args = [sys.executable, 'tools/',
'-mrelease', '-rnone', '-cdart2analyzer'] + standard_args
# TODO(27065): Restore Dartium testing once it works on again.
for runtime in JS_RUNTIMES[bot_info.system]:
with BuildStep('dart2js-%s%s' % (runtime, suffix), swallow_error=True):
test_args = [sys.executable, 'tools/',
'-mrelease', '-r%s' % runtime, '-cdart2js', '-j4',
args = test_args + standard_args
_RunWithXvfb(bot_info, args)
def FillMagicMarkers(v, replacements):
def replace(match):
word =
if not word in replacements:
raise Exception("Unknown magic marker %s. Known mappings are: %s" %
(word, replacements))
return replacements[word]
return re.sub(r"\$(\w+)", replace, v)
def RunTestRunner(bot_info, test_package, package_path):
package_name = os.path.basename(package_path)
if package_name == '':
# when package_path had a trailing slash
package_name = os.path.basename(os.path.dirname(package_path))
pub = GetPub(bot_info)
extra_env = GetPubEnv(bot_info)
with BuildStep('pub run test', swallow_error=True):
# TODO(nweiz): include dartium here once sdk#23816 is fixed.
platforms = set(['vm', 'chrome', 'firefox'])
if bot_info.system == 'windows':
# TODO(nweiz): remove dartium here once sdk#23816 is fixed.
elif bot_info.system == 'mac':
if 'platforms' in test_package:
platforms = platforms.intersection(set(test_package['platforms']))
with utils.ChangedWorkingDirectory(package_path):
test_args = [pub, 'run', 'test', '--reporter', 'expanded', '--no-color',
'--platform', ','.join(platforms)]
if bot_info.package_name in SERIALIZED_PACKAGES:
# TODO(6): If barback is needed, use --pub-serve option and pub serve test
if test_package.get('barback'): test_args.append('build/test')
_RunWithXvfb(bot_info, test_args, extra_env=extra_env)
def _RunWithXvfb(bot_info, args, **kwargs):
if bot_info.system == 'linux':
args = ['xvfb-run', '-a', '--server-args=-screen 0 1024x768x24'] + args
RunProcess(args, **kwargs)
# Runs the script given by test_config.get_config if it exists, does nothing
# otherwise.
# Returns `True` if the script was run.
def RunCustomScript(test_config):
custom_script = test_config.get_custom_script()
if custom_script:
command_string = FillMagicMarkers(custom_script, test_config.replacements)
with BuildStep('Running custom script'):
args = shlex.split(command_string, posix=False)
print 'Running command: %s' % args
exit_code =
if exit_code != 0:
print "Custom script failed"
return True
return False
def RunDefaultScript(bot_info, test_config, copy_path):
print "No custom script found, running default steps."
print 'Running testing in copy of package in %s' % copy_path
RunPubUpgrade(bot_info, copy_path)
test_package = test_config.get_test_package()
if test_package is None or test_package.get('barback'):
RunPubBuild(bot_info, copy_path, 'web')
RunPubBuild(bot_info, copy_path, 'test', 'debug')
if test_package is not None:
print 'Running the test package runner'
RunTestRunner(bot_info, test_package, copy_path)
print 'Running tests manually'
RunPackageTesting(bot_info, copy_path, 'test')
# TODO(6): Packages that need barback should use the test package runner,
# instead of trying to run from the build/test directory.
RunPackageTesting(bot_info, copy_path, 'build/test')
def RunHooks(hooks, section_name, replacements):
for name, command in hooks.iteritems():
command = FillMagicMarkers(command, replacements)
with BuildStep('%s: %s' % (section_name, name), swallow_error=True):
RunProcess(command, shell=True)
def RunPrePubUpgradeHooks(test_config):
RunHooks(test_config.get_pre_pub_upgrade_hooks(), "Pre pub upgrade hooks",
def RunPrePubBuildHooks(test_config):
RunHooks(test_config.get_pre_pub_build_hooks(), "Pre pub build hooks",
def RunPostPubBuildHooks(test_config):
RunHooks(test_config.get_post_pub_build_hooks(), "Pre pub build hooks",
def RunPreTestHooks(test_config):
RunHooks(test_config.get_pre_test_hooks(), "Pre test hooks",
def RunPostTestHooks(test_config):
RunHooks(test_config.get_post_test_hooks(), "Post test hooks",
def main():
bot_info = GetBotInfo()
print 'Bot info: %s' % bot_info
copy_path = GetPackageCopy(bot_info)
config_file = os.path.join(copy_path, '.test_config')
test_config = config_parser.ConfigParser(config_file)
test_config.replacements = {
'dart': utils.CheckedInSdkExecutable(),
'project_root': copy_path,
'python': sys.executable
RunCustomScript(test_config) or \
RunDefaultScript(bot_info, test_config, copy_path)
if __name__ == '__main__':