blob: 0b8270111ba4d9865b1dfcbe3c32b9e3f73800db [file] [log] [blame]
#!/usr/bin/env python3
#
# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
# for details. All rights reserved. Use of this source code is governed by a
# BSD-style license that can be found in the LICENSE file.
import hashlib
import imp
import os
import subprocess
import sys
DART_DIR = os.path.abspath(
os.path.normpath(os.path.join(__file__, '..', '..', '..')))
def GetUtils():
'''Dynamically load the tools/utils.py python module.'''
return imp.load_source('utils', os.path.join(DART_DIR, 'tools', 'utils.py'))
SYSTEM_RENAMES = {
'win32': 'windows',
'windows': 'windows',
'win': 'windows',
'linux': 'linux',
'linux2': 'linux',
'lucid32': 'linux',
'lucid64': 'linux',
'darwin': 'macos',
'mac': 'macos',
'macos': 'macos',
}
ARCH_RENAMES = {
'ia32': 'ia32',
'x64': 'x64',
'arm': 'arm',
'arm64': 'arm64',
'riscv64': 'riscv64',
}
SYSTEM_TO_CIPD = {
'win32': 'windows',
'windows': 'windows',
'linux': 'linux',
'macos': 'mac',
}
ARCH_TO_CIPD = {
'ia32': '386',
'x64': 'amd64',
'arm': 'arm6l',
'arm64': 'arm64',
'riscv64': 'riscv64',
}
class Channel(object):
BETA = 'beta'
BLEEDING_EDGE = 'be'
DEV = 'dev'
STABLE = 'stable'
TRY = 'try'
INTEGRATION = 'integration'
ALL_CHANNELS = [BETA, BLEEDING_EDGE, DEV, STABLE, TRY, INTEGRATION]
class ReleaseType(object):
RAW = 'raw'
SIGNED = 'signed'
RELEASE = 'release'
ALL_TYPES = [RAW, SIGNED, RELEASE]
class Mode(object):
RELEASE = 'release'
DEBUG = 'debug'
ALL_MODES = [RELEASE, DEBUG]
class GCSNamer(object):
"""
This class is used for naming objects in our "gs://dart-archive/"
GoogleCloudStorage bucket. It's structure is as follows:
For every (channel,revision,release-type) tuple we have a base path:
gs://dart-archive/channels/{be,beta,dev,stable,try,integration}
/{raw,signed,release}/{revision,latest}/
Under every base path, the following structure is used:
- /VERSION
- /api-docs/dartdocs-gen-api.zip
- /sdk/dartsdk-{linux,macos,windows}-{ia32,x64}-release.zip
"""
def __init__(self,
channel=Channel.BLEEDING_EDGE,
release_type=ReleaseType.RAW,
internal=False):
assert channel in Channel.ALL_CHANNELS
assert release_type in ReleaseType.ALL_TYPES
self.channel = channel
self.release_type = release_type
if internal:
self.bucket = 'gs://dart-archive-internal'
else:
self.bucket = 'gs://dart-archive'
# Functions for quering complete gs:// filepaths
def version_filepath(self, revision):
return '%s/channels/%s/%s/%s/VERSION' % (self.bucket, self.channel,
self.release_type, revision)
def sdk_zipfilepath(self, revision, system, arch, mode):
return '/'.join([
self.sdk_directory(revision),
self.sdk_zipfilename(system, arch, mode)
])
def unstripped_filepath(self, revision, system, arch):
return '/'.join([
self._variant_directory('unstripped', revision), system, arch,
self.unstripped_filename(system)
])
def apidocs_zipfilepath(self, revision):
return '/'.join(
[self.apidocs_directory(revision),
self.dartdocs_zipfilename()])
# Functions for querying gs:// directories
def base_directory(self, revision):
return '%s/channels/%s/%s/%s' % (self.bucket, self.channel,
self.release_type, revision)
def sdk_directory(self, revision):
return self._variant_directory('sdk', revision)
def linux_packages_directory(self, revision):
return '/'.join([self._variant_directory('linux_packages', revision)])
def src_directory(self, revision):
return self._variant_directory('src', revision)
def apidocs_directory(self, revision):
return self._variant_directory('api-docs', revision)
def misc_directory(self, revision):
return self._variant_directory('misc', revision)
def _variant_directory(self, name, revision):
return '%s/%s' % (self.base_directory(revision), name)
# Functions for quering filenames
def dartdocs_zipfilename(self):
return 'dartdocs-gen-api.zip'
def sdk_zipfilename(self, system, arch, mode):
assert mode in Mode.ALL_MODES
return 'dartsdk-%s-%s-%s.zip' % (SYSTEM_RENAMES[system],
ARCH_RENAMES[arch], mode)
def unstripped_filename(self, system):
return 'dart.exe' if system.startswith('win') else 'dart'
class GCSNamerApiDocs(object):
def __init__(self, channel=Channel.BLEEDING_EDGE):
assert channel in Channel.ALL_CHANNELS
self.channel = channel
self.bucket = 'gs://dartlang-api-docs'
def dartdocs_dirpath(self, revision):
assert len('%s' % revision) > 0
if self.channel == Channel.BLEEDING_EDGE:
return '%s/gen-dartdocs/builds/%s' % (self.bucket, revision)
return '%s/gen-dartdocs/%s/%s' % (self.bucket, self.channel, revision)
def docs_latestpath(self, revision):
assert len('%s' % revision) > 0
return '%s/channels/%s/latest.txt' % (self.bucket, self.channel)
def run(command, env=None, shell=False, throw_on_error=True):
print("Running command: ", command)
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=env,
shell=shell,
universal_newlines=True)
(stdout, stderr) = p.communicate()
if throw_on_error and p.returncode != 0:
print("Failed to execute '%s'. Exit code: %s." %
(command, p.returncode),
file=sys.stderr)
print("stdout: ", stdout, file=sys.stderr)
print("stderr: ", stderr, file=sys.stderr)
raise Exception("Failed to execute %s." % command)
return (stdout, stderr, p.returncode)
class GSUtil(object):
GSUTIL_PATH = None
USE_DART_REPO_VERSION = False
def _layzCalculateGSUtilPath(self):
if not GSUtil.GSUTIL_PATH:
dart_gsutil = os.path.join(DART_DIR, 'third_party', 'gsutil',
'gsutil')
if os.path.isfile(dart_gsutil):
GSUtil.GSUTIL_PATH = dart_gsutil
elif GSUtil.USE_DART_REPO_VERSION:
raise Exception("Dart repository version of gsutil required, "
"but not found.")
else:
# We did not find gsutil, look in path
possible_locations = list(os.environ['PATH'].split(os.pathsep))
for directory in possible_locations:
location = os.path.join(directory, 'gsutil')
if os.path.isfile(location):
GSUtil.GSUTIL_PATH = location
break
assert GSUtil.GSUTIL_PATH
def execute(self, gsutil_args):
self._layzCalculateGSUtilPath()
gsutil_command = [sys.executable, GSUtil.GSUTIL_PATH]
return run(gsutil_command + gsutil_args)
def upload(self,
local_path,
remote_path,
recursive=False,
multithread=False):
assert remote_path.startswith('gs://')
if multithread:
args = ['-m', 'cp']
else:
args = ['cp']
if recursive:
args += ['-R']
args += [local_path, remote_path]
self.execute(args)
def cat(self, remote_path):
assert remote_path.startswith('gs://')
args = ['cat', remote_path]
(stdout, _, _) = self.execute(args)
return stdout
def setGroupReadACL(self, remote_path, group):
args = ['acl', 'ch', '-g', '%s:R' % group, remote_path]
self.execute(args)
def setContentType(self, remote_path, content_type):
args = ['setmeta', '-h', 'Content-Type:%s' % content_type, remote_path]
self.execute(args)
def remove(self, remote_path, recursive=False):
assert remote_path.startswith('gs://')
args = ['rm']
if recursive:
args += ['-R']
args += [remote_path]
self.execute(args)
def CalculateMD5Checksum(filename):
"""Calculate the MD5 checksum for filename."""
md5 = hashlib.md5()
with open(filename, 'rb') as f:
data = f.read(65536)
while len(data) > 0:
md5.update(data)
data = f.read(65536)
return md5.hexdigest()
def CalculateSha256Checksum(filename):
"""Calculate the sha256 checksum for filename."""
sha = hashlib.sha256()
with open(filename, 'rb') as f:
data = f.read(65536)
while len(data) > 0:
sha.update(data)
data = f.read(65536)
return sha.hexdigest()
def CreateMD5ChecksumFile(filename, mangled_filename=None):
"""Create and upload an MD5 checksum file for filename."""
if not mangled_filename:
mangled_filename = os.path.basename(filename)
checksum = CalculateMD5Checksum(filename)
checksum_filename = '%s.md5sum' % filename
with open(checksum_filename, 'w') as f:
f.write('%s *%s' % (checksum, mangled_filename))
print("MD5 checksum of %s is %s" % (filename, checksum))
return checksum_filename
def CreateSha256ChecksumFile(filename, mangled_filename=None):
"""Create and upload an sha256 checksum file for filename."""
if not mangled_filename:
mangled_filename = os.path.basename(filename)
checksum = CalculateSha256Checksum(filename)
checksum_filename = '%s.sha256sum' % filename
with open(checksum_filename, 'w') as f:
f.write('%s *%s' % (checksum, mangled_filename))
print("SHA256 checksum of %s is %s" % (filename, checksum))
return checksum_filename
def GetChannelFromName(name):
"""Get the channel from the name. Bleeding edge builders don't
have a suffix."""
channel_name = name.split('-').pop()
if channel_name in Channel.ALL_CHANNELS:
return channel_name
return Channel.BLEEDING_EDGE
def GetSystemFromName(name):
"""Get the system from the name."""
for part in name.split('-'):
if part in SYSTEM_RENAMES: return SYSTEM_RENAMES[part]
raise ValueError(
"Bot name '{}' does not have a system name in it.".format(name))