[infra] Add versionchecker and debianpackage to test matrix
* Run Debian package creation in a Docker container.
* Upload archives from Docker host (the container lacks credentials).
* Remove BUILDBOT_BUILDERNAME environment variables from test matrix:
They are overwritten in the recipe.
Change-Id: I90d7747e64231bdb745c85030a9cecf256adbaf0
Reviewed-on: https://dart-review.googlesource.com/c/sdk/+/94542
Reviewed-by: William Hesse <whesse@google.com>
diff --git a/tools/bots/linux_distribution_support.py b/tools/bots/linux_distribution_support.py
index 7e86e8c..28c26c7 100644
--- a/tools/bots/linux_distribution_support.py
+++ b/tools/bots/linux_distribution_support.py
@@ -35,33 +35,14 @@
return None
return bot.BuildInfo('none', 'none', 'release', 'linux')
-def ArchiveArtifacts(tarfile, builddir, channel):
- namer = bot_utils.GCSNamer(channel=channel)
- gsutil = bot_utils.GSUtil()
- revision = utils.GetArchiveVersion()
- # Archive the src tar to the src dir
- remote_tarfile = '/'.join([namer.src_directory(revision),
- os.path.basename(tarfile)])
- gsutil.upload(tarfile, remote_tarfile, public=True)
- # Archive all files except the tar file to the linux packages dir
- for entry in os.listdir(builddir):
- full_path = os.path.join(builddir, entry)
- # We expect a flat structure, not subdirectories
- assert(os.path.isfile(full_path))
- if full_path != tarfile:
- package_dir = namer.linux_packages_directory(revision)
- remote_file = '/'.join([package_dir,
- os.path.basename(entry)])
- gsutil.upload(full_path, remote_file, public=True)
-
def InstallFromDep(builddir):
for entry in os.listdir(builddir):
if entry.endswith("_amd64.deb"):
path = os.path.join(builddir, entry)
- Run(['sudo', 'dpkg', '-i', path])
+ Run(['dpkg', '-i', path])
def UninstallDart():
- Run(['sudo', 'dpkg', '-r', 'dart'])
+ Run(['dpkg', '-r', 'dart'])
def CreateDartTestFile(tempdir):
filename = os.path.join(tempdir, 'test.dart')
@@ -162,13 +143,6 @@
UninstallDart()
TestInstallation(assume_installed=False)
- with bot.BuildStep('Upload artifacts'):
- bot_name, _ = bot.GetBotName()
- channel = bot_utils.GetChannelFromName(bot_name)
- if channel != bot_utils.Channel.BLEEDING_EDGE:
- ArchiveArtifacts(tarfile, builddir, channel)
- else:
- print 'Not uploading artifacts on bleeding edge'
if __name__ == '__main__':
# We pass in None for build_step to avoid building the sdk.
diff --git a/tools/bots/test_matrix.json b/tools/bots/test_matrix.json
index 4efcdb6..0dc5fc2 100644
--- a/tools/bots/test_matrix.json
+++ b/tools/bots/test_matrix.json
@@ -1246,16 +1246,12 @@
{
"name": "build dart",
"script": "tools/bots/dart_sdk.py",
- "arguments": [],
- "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-linux"}
-
+ "arguments": []
},
{
"name": "build api docs",
"script": "tools/bots/dart_sdk.py",
- "arguments": [ "api_docs" ],
- "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-linux"}
-
+ "arguments": [ "api_docs" ]
},
{
"name": "upload abi dills",
@@ -1263,8 +1259,7 @@
"arguments": [
"tools/VERSION",
"out/ReleaseX64"
- ],
- "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-linux"}
+ ]
}
]
},
@@ -1277,9 +1272,7 @@
{
"name": "build dart",
"script": "tools/bots/dart_sdk.py",
- "arguments": [],
- "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-mac"}
-
+ "arguments": []
}
]
},
@@ -1292,9 +1285,38 @@
{
"name": "build dart",
"script": "tools/bots/dart_sdk.py",
- "arguments": [],
- "environment": {"BUILDBOT_BUILDERNAME": "dart-sdk-win"}
-
+ "arguments": []
+ }
+ ]
+ },
+ {
+ "builders": ["debianpackage-linux"],
+ "meta": {
+ "description": "This configuration is used by the debianpackage-builder."
+ },
+ "steps": [
+ {
+ "name": "build debian package",
+ "script": "tools/run_debian_build.sh",
+ "arguments": []
+ },
+ {
+ "name": "upload debian packages",
+ "script": "tools/bots/upload_debian_packages.py",
+ "arguments": []
+ }
+ ]
+ },
+ {
+ "builders": ["versionchecker-linux"],
+ "meta": {
+ "description": "This configuration is used by the versionchecker-builder."
+ },
+ "steps": [
+ {
+ "name": "check version",
+ "script": "tools/bots/version_checker.py",
+ "arguments": []
}
]
},
diff --git a/tools/bots/upload_debian_packages.py b/tools/bots/upload_debian_packages.py
new file mode 100755
index 0000000..04750c0
--- /dev/null
+++ b/tools/bots/upload_debian_packages.py
@@ -0,0 +1,53 @@
+#!/usr/bin/python
+
+# Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+
+
+import os
+
+
+import bot
+import bot_utils
+
+
+utils = bot_utils.GetUtils()
+
+
+HOST_OS = utils.GuessOS()
+
+
+def ArchiveArtifacts(tarfile, builddir, channel):
+ namer = bot_utils.GCSNamer(channel=channel)
+ gsutil = bot_utils.GSUtil()
+ revision = utils.GetArchiveVersion()
+ # Archive the src tar to the src dir
+ remote_tarfile = '/'.join([namer.src_directory(revision),
+ os.path.basename(tarfile)])
+ gsutil.upload(tarfile, remote_tarfile, public=True)
+ # Archive all files except the tar file to the linux packages dir
+ for entry in os.listdir(builddir):
+ full_path = os.path.join(builddir, entry)
+ # We expect a flat structure, not subdirectories
+ assert(os.path.isfile(full_path))
+ if full_path != tarfile:
+ package_dir = namer.linux_packages_directory(revision)
+ remote_file = '/'.join([package_dir,
+ os.path.basename(entry)])
+ gsutil.upload(full_path, remote_file, public=True)
+
+
+if __name__ == '__main__':
+ bot_name, _ = bot.GetBotName()
+ channel = bot_utils.GetChannelFromName(bot_name)
+ if channel != bot_utils.Channel.BLEEDING_EDGE:
+ builddir = os.path.join(bot_utils.DART_DIR,
+ utils.GetBuildDir(HOST_OS),
+ 'src_and_installation')
+ version = utils.GetVersion()
+ tarfilename = 'dart-%s.tar.gz' % version
+ tarfile = os.path.join(builddir, tarfilename)
+ ArchiveArtifacts(tarfile, builddir, channel)
+ else:
+ print 'Not uploading artifacts on bleeding edge'
diff --git a/tools/run_debian_build.sh b/tools/run_debian_build.sh
new file mode 100755
index 0000000..66e7d5d
--- /dev/null
+++ b/tools/run_debian_build.sh
@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+# Copyright (c) 2019, the Dart project authors. Please see the AUTHORS file
+# for details. All rights reserved. Use of this source code is governed by a
+# BSD-style license that can be found in the LICENSE file.
+set -x
+
+ninja=$(which ninja)
+depot_tools=$(dirname $ninja)
+cmd="apt-get update && apt-get -y install build-essential debhelper git python\
+ && PATH=\"$depot_tools:\$PATH\"\
+ python tools/bots/linux_distribution_support.py"
+image="launcher.gcr.io/google/debian8:latest"
+docker run -e BUILDBOT_BUILDERNAME -v $depot_tools:$depot_tools\
+ -v `pwd`:`pwd` -w `pwd` -i --rm $image bash -c "$cmd"
diff --git a/tools/utils.py b/tools/utils.py
index a897eb1..a6db548 100644
--- a/tools/utils.py
+++ b/tools/utils.py
@@ -458,12 +458,12 @@
return fd.read()
except:
pass
-
- p = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:%H'],
+ p = subprocess.Popen(['git', 'rev-parse', 'HEAD'],
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT, shell=IsWindows(),
cwd = DART_DIR)
output, _ = p.communicate()
+ output = output.strip()
# We expect a full git hash
if len(output) != 40:
print "Warning: could not parse git commit, output was %s" % output
@@ -472,11 +472,12 @@
def GetShortGitHash():
- p = subprocess.Popen(['git', 'log', '-n', '1', '--pretty=format:%h'],
+ p = subprocess.Popen(['git', '--short', 'rev-parse', 'HEAD'],
stdout = subprocess.PIPE,
stderr = subprocess.STDOUT, shell=IsWindows(),
cwd = DART_DIR)
output, _ = p.communicate()
+ output = output.strip()
if p.wait() != 0:
return None
return output