blob: 50f3978d15992b34c99ae251102d388d95a1cba6 [file] [log] [blame]
#!/usr/bin/env python
#
# Copyright (c) 2015, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import os
import os.path
import shutil
import sys
import subprocess
import bot
import bot_utils
utils = bot_utils.GetUtils()
BUILD_OS = utils.GuessOS()
BUILD_ARCHITECTURE = utils.GuessArchitecture()
(bot_name, _) = bot.GetBotName()
CHANNEL = bot_utils.GetChannelFromName(bot_name)
def BuildArchitectures():
if BUILD_OS == 'linux':
return ['ia32', 'x64', 'arm', 'arm64']
else:
return ['ia32', 'x64']
def BuildSDK():
with bot.BuildStep('Build SDK'):
if BUILD_OS == 'linux':
sysroot_env = dict(os.environ)
sysroot_env['DART_USE_WHEEZY'] = '1'
Run([sys.executable, './tools/generate_buildfiles.py'], env=sysroot_env)
for arch in BuildArchitectures():
Run([sys.executable, './tools/build.py', '--mode=release',
'--arch=' + arch, 'create_sdk'])
def BuildDartdocAPIDocs(dirname):
dart_sdk = os.path.join(bot_utils.DART_DIR,
utils.GetBuildRoot(BUILD_OS, 'release',
BUILD_ARCHITECTURE),
'dart-sdk')
dart_exe = os.path.join(dart_sdk, 'bin', 'dart')
dartdoc_dart = os.path.join(bot_utils.DART_DIR,
'third_party', 'pkg' , 'dartdoc' , 'bin' ,
'dartdoc.dart')
footer_file = os.path.join(bot_utils.DART_DIR,
'tools', 'bots', 'dartdoc_footer.html')
url = 'https://api.dartlang.org/stable'
with bot.BuildStep('Build API docs by dartdoc'):
bot_utils.run([dart_exe, dartdoc_dart,
'--sdk-docs','--output', dirname, '--footer' , footer_file,
'--rel-canonical-prefix=' + url])
def CreateUploadVersionFile():
file_path = os.path.join(bot_utils.DART_DIR,
utils.GetBuildRoot(BUILD_OS, 'release',
BUILD_ARCHITECTURE),
'VERSION')
with open(file_path, 'w') as fd:
fd.write(utils.GetVersionFileContent())
DartArchiveUploadVersionFile(file_path)
def DartArchiveUploadVersionFile(version_file):
namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
revision = utils.GetArchiveVersion()
for revision in [revision, 'latest']:
destination = namer.version_filepath(revision)
DartArchiveFile(version_file, destination, checksum_files=False)
def CreateUploadSDKZips():
with bot.BuildStep('Create and upload sdk zips'):
for arch in BuildArchitectures():
sdk_path = os.path.join(bot_utils.DART_DIR,
utils.GetBuildRoot(BUILD_OS, 'release', arch),
'dart-sdk')
sdk_zip = os.path.join(bot_utils.DART_DIR,
utils.GetBuildRoot(BUILD_OS, 'release', arch),
'dartsdk-%s-%s.zip' % (BUILD_OS, arch))
FileDelete(sdk_zip)
CreateZip(sdk_path, sdk_zip)
DartArchiveUploadSDKs(BUILD_OS, arch, sdk_zip)
def DartArchiveUploadSDKs(system, arch, sdk_zip):
namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
git_number = utils.GetArchiveVersion()
git_hash = 'hash/%s' % utils.GetGitRevision()
for revision in [git_number, git_hash, 'latest']:
path = namer.sdk_zipfilepath(revision, system, arch, 'release')
DartArchiveFile(sdk_zip, path, checksum_files=True)
def DartArchiveUnstrippedBinaries():
namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
revision = utils.GetArchiveVersion()
binary = namer.unstripped_filename(BUILD_OS)
for arch in BuildArchitectures():
binary = os.path.join(bot_utils.DART_DIR,
utils.GetBuildRoot(BUILD_OS, 'release', arch),
binary)
gs_path = namer.unstripped_filepath(revision, BUILD_OS, arch)
DartArchiveFile(binary, gs_path)
def CreateUploadSDK():
BuildSDK()
CreateUploadSDKZips()
DartArchiveUnstrippedBinaries()
def CreateUploadAPIDocs():
dartdoc_dir = os.path.join(bot_utils.DART_DIR,
utils.GetBuildRoot(BUILD_OS, 'release',
BUILD_ARCHITECTURE),
'gen-dartdocs')
dartdoc_zip = os.path.join(bot_utils.DART_DIR,
utils.GetBuildRoot(BUILD_OS, 'release',
BUILD_ARCHITECTURE),
'dartdocs-api.zip')
if CHANNEL == 'try':
BuildDartdocAPIDocs(dartdoc_dir)
else:
UploadApiLatestFile()
BuildDartdocAPIDocs(dartdoc_dir)
UploadDartdocApiDocs(dartdoc_dir)
CreateZip(dartdoc_dir, dartdoc_zip)
DartArchiveUploadDartdocAPIDocs(dartdoc_zip)
def DartArchiveUploadDartdocAPIDocs(api_zip):
namer = bot_utils.GCSNamer(CHANNEL, bot_utils.ReleaseType.RAW)
revision = utils.GetArchiveVersion()
for revision in [revision, 'latest']:
destination = (namer.apidocs_directory(revision) + '/' +
namer.dartdocs_zipfilename())
DartArchiveFile(api_zip, destination, checksum_files=False)
def UploadDartdocApiDocs(dir_name):
apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL)
revision = utils.GetArchiveVersion()
dartdocs_destination_gcsdir = apidocs_namer.dartdocs_dirpath(revision)
# Return early if the documents have already been uploaded.
# This can happen if a build was forced, or a commit had no changes in the
# dart repository (e.g. DEPS file update).
if GsutilExists(dartdocs_destination_gcsdir):
print ("Not uploading api docs, since %s is already present."
% dartdocs_destination_gcsdir)
return
# Upload everything inside the built apidocs directory.
gsutil = bot_utils.GSUtil()
gsutil.upload(dir_name, dartdocs_destination_gcsdir, recursive=True,
public=True, multithread=True)
def UploadApiLatestFile():
apidocs_namer = bot_utils.GCSNamerApiDocs(CHANNEL)
revision = utils.GetArchiveVersion()
apidocs_destination_latestfile = apidocs_namer.docs_latestpath(revision)
# Update latest.txt to contain the newest revision.
with utils.TempDir('latest_file') as temp_dir:
latest_file = os.path.join(temp_dir, 'latest.txt')
with open(latest_file, 'w') as f:
f.write('%s' % revision)
DartArchiveFile(latest_file, apidocs_destination_latestfile)
def GsutilExists(gsu_path):
# This is a little hackish, but it is basically a one off doing very
# specialized check that we don't use elsewhere.
gsutilTool = os.path.join(bot_utils.DART_DIR,
'third_party', 'gsutil', 'gsutil')
(_, stderr, returncode) = bot_utils.run(
[gsutilTool, 'ls', gsu_path],
throw_on_error=False)
# If the returncode is nonzero and we can find a specific error message,
# we know there are no objects with a prefix of [gsu_path].
missing = (returncode and 'CommandException: No such object' in stderr)
# Either the returncode has to be zero or the object must be missing,
# otherwise throw an exception.
if not missing and returncode:
raise Exception("Failed to determine whether %s exists" % gsu_path)
return not missing
def CreateZip(directory, target_file):
if 'win' in BUILD_OS:
CreateZipWindows(directory, target_file)
else:
CreateZipPosix(directory, target_file)
def CreateZipPosix(directory, target_file):
with utils.ChangedWorkingDirectory(os.path.dirname(directory)):
command = ['zip', '-yrq9', target_file, os.path.basename(directory)]
Run(command)
def CreateZipWindows(directory, target_file):
with utils.ChangedWorkingDirectory(os.path.dirname(directory)):
zip_win = os.path.join(bot_utils.DART_DIR, 'third_party', '7zip', '7za')
command = [zip_win, 'a', '-tzip', target_file, os.path.basename(directory)]
Run(command)
def FileDelete(f):
if os.path.exists(f):
os.remove(f)
def DartArchiveFile(local_path, remote_path, checksum_files=False):
gsutil = bot_utils.GSUtil()
gsutil.upload(local_path, remote_path, public=True)
if checksum_files:
# 'local_path' may have a different filename than 'remote_path'. So we need
# to make sure the *.md5sum file contains the correct name.
assert '/' in remote_path and not remote_path.endswith('/')
mangled_filename = remote_path[remote_path.rfind('/') + 1:]
local_md5sum = bot_utils.CreateMD5ChecksumFile(local_path,
mangled_filename)
gsutil.upload(local_md5sum, remote_path + '.md5sum', public=True)
local_sha256 = bot_utils.CreateSha256ChecksumFile(local_path,
mangled_filename)
gsutil.upload(local_sha256, remote_path + '.sha256sum', public=True)
def Run(command, env=None):
print "Running %s" % ' '.join(command)
print "Environment %s" % env
return bot.RunProcess(command, env=env)
if __name__ == '__main__':
if len(sys.argv) > 1 and sys.argv[1] == 'api_docs':
if BUILD_OS == 'linux':
CreateUploadAPIDocs()
else:
# We always clobber the bot, to make sure releases are build from scratch
force = CHANNEL != bot_utils.Channel.BLEEDING_EDGE
bot.Clobber(force=force)
CreateUploadSDK()
if BUILD_OS == 'linux':
CreateUploadVersionFile()