[dart_ci] Add recipe to build CIPD package with executable for dart_ci scripts
Change-Id: I8652bfd241c5b3f94a61c5f33e666ba16985081e
Reviewed-on: https://dart-review.googlesource.com/c/recipes/+/203504
Reviewed-by: Alexander Thomas <athom@google.com>
Commit-Queue: Karl Klose <karlklose@google.com>
diff --git a/README.recipes.md b/README.recipes.md
index e318219..82c6b92 100644
--- a/README.recipes.md
+++ b/README.recipes.md
@@ -23,6 +23,7 @@
* [dart/linearize](#recipes-dart_linearize)
* [dart/neo](#recipes-dart_neo)
* [dart/package_co19](#recipes-dart_package_co19)
+ * [dart/package_dart_ci](#recipes-dart_package_dart_ci)
* [dart:examples/example](#recipes-dart_examples_example)
* [dart:examples/example-get_secret](#recipes-dart_examples_example-get_secret)
* [presubmit/presubmit](#recipes-presubmit_presubmit)
@@ -242,6 +243,11 @@
[DEPS](/recipes/dart/package_co19.py#8): [depot\_tools/git][depot_tools/recipe_modules/git], [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/cipd][recipe_engine/recipe_modules/cipd], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/step][recipe_engine/recipe_modules/step]
— **def [RunSteps](/recipes/dart/package_co19.py#18)(api):**
+### *recipes* / [dart/package\_dart\_ci](/recipes/dart/package_dart_ci.py)
+
+[DEPS](/recipes/dart/package_dart_ci.py#10): [depot\_tools/git][depot_tools/recipe_modules/git], [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/cipd][recipe_engine/recipe_modules/cipd], [recipe\_engine/context][recipe_engine/recipe_modules/context], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/step][recipe_engine/recipe_modules/step]
+
+— **def [RunSteps](/recipes/dart/package_dart_ci.py#20)(api):**
### *recipes* / [dart:examples/example](/recipe_modules/dart/examples/example.py)
[DEPS](/recipe_modules/dart/examples/example.py#21): [build](#recipe_modules-build), [dart](#recipe_modules-dart), [depot\_tools/tryserver][depot_tools/recipe_modules/tryserver], [recipe\_engine/buildbucket][recipe_engine/recipe_modules/buildbucket], [recipe\_engine/file][recipe_engine/recipe_modules/file], [recipe\_engine/json][recipe_engine/recipe_modules/json], [recipe\_engine/path][recipe_engine/recipe_modules/path], [recipe\_engine/platform][recipe_engine/recipe_modules/platform], [recipe\_engine/properties][recipe_engine/recipe_modules/properties], [recipe\_engine/raw\_io][recipe_engine/recipe_modules/raw_io], [recipe\_engine/runtime][recipe_engine/recipe_modules/runtime], [recipe\_engine/step][recipe_engine/recipe_modules/step], [recipe\_engine/swarming][recipe_engine/recipe_modules/swarming]
diff --git a/recipes/dart/package_dart_ci.expected/basic-with-revision.json b/recipes/dart/package_dart_ci.expected/basic-with-revision.json
new file mode 100644
index 0000000..4df7c82
--- /dev/null
+++ b/recipes/dart/package_dart_ci.expected/basic-with-revision.json
@@ -0,0 +1,195 @@
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
+ "--path",
+ "[START_DIR]/dart_ci",
+ "--url",
+ "https://dart.googlesource.com/dart_ci"
+ ],
+ "name": "git setup"
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "origin",
+ "refs/changes/20/203220/9",
+ "--progress"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "env": {
+ "PATH": "RECIPE_REPO[depot_tools]:<PATH>"
+ },
+ "infra_step": true,
+ "name": "git fetch"
+ },
+ {
+ "cmd": [
+ "git",
+ "checkout",
+ "-f",
+ "FETCH_HEAD"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "git checkout"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "read revision",
+ "~followup_annotations": [
+ "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@",
+ "@@@SET_BUILD_PROPERTY@got_revision@\"deadbeef\"@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "clean",
+ "-f",
+ "-d",
+ "-x"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "git clean"
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "sync"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "submodule sync"
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "submodule update"
+ },
+ {
+ "cmd": [],
+ "name": "install dart/dart-sdk"
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/cipd/dart/dart-sdk/zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C"
+ ],
+ "infra_step": true,
+ "name": "install dart/dart-sdk.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/cipd/dart/dart-sdk/zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C",
+ "-ensure-file",
+ "dart/dart-sdk/${platform} zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "install dart/dart-sdk.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-zysAwdt80NCILRb5\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"dart/dart-sdk/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/cipd/dart/dart-sdk/zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C/dart-sdk/bin/dart",
+ "pub",
+ "get"
+ ],
+ "cwd": "[START_DIR]/dart_ci/builder",
+ "env": {
+ "PUB_CACHE": "[CLEANUP]/tmp_tmp_2"
+ },
+ "name": "pub get"
+ },
+ {
+ "cmd": [
+ "[CACHE]/cipd/dart/dart-sdk/zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C/dart-sdk/bin/dart",
+ "compile",
+ "exe",
+ "[START_DIR]/dart_ci/builder/bin/update_results_database.dart",
+ "-o",
+ "[CLEANUP]/tmp_tmp_1/update_results_database"
+ ],
+ "cwd": "[START_DIR]/dart_ci/builder",
+ "env": {
+ "PUB_CACHE": "[CLEANUP]/tmp_tmp_2"
+ },
+ "name": "compile script"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "create",
+ "-pkg-def",
+ "{\"data\": [{\"file\": \"update_results_database\"}], \"install_mode\": \"copy\", \"package\": \"dart/ci/builder_scripts/${platform}\", \"root\": \"[CLEANUP]/tmp_tmp_1\"}",
+ "-hash-algo",
+ "sha256",
+ "-tag",
+ "git_revision:deadbeef",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "create dart/ci/builder_scripts/${platform}",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"dart/ci/builder_scripts/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LINK@40-chars-fake-of-the-package-instance_id@https://chrome-infra-packages.appspot.com/p/dart/ci/builder_scripts/resolved-platform/+/40-chars-fake-of-the-package-instance_id@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipes/dart/package_dart_ci.expected/basic.json b/recipes/dart/package_dart_ci.expected/basic.json
new file mode 100644
index 0000000..f8bf35e
--- /dev/null
+++ b/recipes/dart/package_dart_ci.expected/basic.json
@@ -0,0 +1,194 @@
+[
+ {
+ "cmd": [
+ "python",
+ "-u",
+ "RECIPE_MODULE[depot_tools::git]/resources/git_setup.py",
+ "--path",
+ "[START_DIR]/dart_ci",
+ "--url",
+ "https://dart.googlesource.com/dart_ci"
+ ],
+ "name": "git setup"
+ },
+ {
+ "cmd": [
+ "git",
+ "fetch",
+ "origin",
+ "--progress"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "env": {
+ "PATH": "RECIPE_REPO[depot_tools]:<PATH>"
+ },
+ "infra_step": true,
+ "name": "git fetch"
+ },
+ {
+ "cmd": [
+ "git",
+ "checkout",
+ "-f",
+ "2d72510e447ab60a9728aeea2362d8be2cbd7789"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "git checkout"
+ },
+ {
+ "cmd": [
+ "git",
+ "rev-parse",
+ "HEAD"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "read revision",
+ "~followup_annotations": [
+ "@@@STEP_TEXT@<br/>checked out 'deadbeef'<br/>@@@",
+ "@@@SET_BUILD_PROPERTY@got_revision@\"deadbeef\"@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "git",
+ "clean",
+ "-f",
+ "-d",
+ "-x"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "git clean"
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "sync"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "submodule sync"
+ },
+ {
+ "cmd": [
+ "git",
+ "submodule",
+ "update",
+ "--init",
+ "--recursive"
+ ],
+ "cwd": "[START_DIR]/dart_ci",
+ "infra_step": true,
+ "name": "submodule update"
+ },
+ {
+ "cmd": [],
+ "name": "install dart/dart-sdk"
+ },
+ {
+ "cmd": [
+ "vpython",
+ "-u",
+ "RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
+ "--json-output",
+ "/path/to/tmp/json",
+ "ensure-directory",
+ "--mode",
+ "0777",
+ "[CACHE]/cipd/dart/dart-sdk/zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C"
+ ],
+ "infra_step": true,
+ "name": "install dart/dart-sdk.ensure package directory",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "cipd",
+ "ensure",
+ "-root",
+ "[CACHE]/cipd/dart/dart-sdk/zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C",
+ "-ensure-file",
+ "dart/dart-sdk/${platform} zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C",
+ "-max-threads",
+ "0",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "infra_step": true,
+ "name": "install dart/dart-sdk.ensure_installed",
+ "~followup_annotations": [
+ "@@@STEP_NEST_LEVEL@1@@@",
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"\": [@@@",
+ "@@@STEP_LOG_LINE@json.output@ {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"resolved-instance_id-of-zysAwdt80NCILRb5\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"dart/dart-sdk/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@ ]@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@"
+ ]
+ },
+ {
+ "cmd": [
+ "[CACHE]/cipd/dart/dart-sdk/zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C/dart-sdk/bin/dart",
+ "pub",
+ "get"
+ ],
+ "cwd": "[START_DIR]/dart_ci/builder",
+ "env": {
+ "PUB_CACHE": "[CLEANUP]/tmp_tmp_2"
+ },
+ "name": "pub get"
+ },
+ {
+ "cmd": [
+ "[CACHE]/cipd/dart/dart-sdk/zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C/dart-sdk/bin/dart",
+ "compile",
+ "exe",
+ "[START_DIR]/dart_ci/builder/bin/update_results_database.dart",
+ "-o",
+ "[CLEANUP]/tmp_tmp_1/update_results_database"
+ ],
+ "cwd": "[START_DIR]/dart_ci/builder",
+ "env": {
+ "PUB_CACHE": "[CLEANUP]/tmp_tmp_2"
+ },
+ "name": "compile script"
+ },
+ {
+ "cmd": [
+ "cipd",
+ "create",
+ "-pkg-def",
+ "{\"data\": [{\"file\": \"update_results_database\"}], \"install_mode\": \"copy\", \"package\": \"dart/ci/builder_scripts/${platform}\", \"root\": \"[CLEANUP]/tmp_tmp_1\"}",
+ "-hash-algo",
+ "sha256",
+ "-tag",
+ "git_revision:deadbeef",
+ "-json-output",
+ "/path/to/tmp/json"
+ ],
+ "name": "create dart/ci/builder_scripts/${platform}",
+ "~followup_annotations": [
+ "@@@STEP_LOG_LINE@json.output@{@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"result\": {@@@",
+ "@@@STEP_LOG_LINE@json.output@ \"instance_id\": \"40-chars-fake-of-the-package-instance_id\", @@@",
+ "@@@STEP_LOG_LINE@json.output@ \"package\": \"dart/ci/builder_scripts/resolved-platform\"@@@",
+ "@@@STEP_LOG_LINE@json.output@ }@@@",
+ "@@@STEP_LOG_LINE@json.output@}@@@",
+ "@@@STEP_LOG_END@json.output@@@",
+ "@@@STEP_LINK@40-chars-fake-of-the-package-instance_id@https://chrome-infra-packages.appspot.com/p/dart/ci/builder_scripts/resolved-platform/+/40-chars-fake-of-the-package-instance_id@@@"
+ ]
+ },
+ {
+ "name": "$result"
+ }
+]
\ No newline at end of file
diff --git a/recipes/dart/package_dart_ci.py b/recipes/dart/package_dart_ci.py
new file mode 100644
index 0000000..45e4f2a
--- /dev/null
+++ b/recipes/dart/package_dart_ci.py
@@ -0,0 +1,66 @@
+# Copyright 2021 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+# Compiles platform specific executables for the scripts in dart_ci/builder
+# and uploads them as CIPD package 'dart/ci/builder_scripts/${platform}'.
+
+from recipe_engine.post_process import Filter
+
+DEPS = [
+ 'depot_tools/git',
+ 'recipe_engine/buildbucket',
+ 'recipe_engine/cipd',
+ 'recipe_engine/context',
+ 'recipe_engine/path',
+ 'recipe_engine/step',
+]
+
+
+def RunSteps(api):
+ # Checkout dart_ci version to build.
+ revision = api.buildbucket.gitiles_commit.id or 'HEAD'
+ got_revision = api.git.checkout(
+ url='https://dart.googlesource.com/dart_ci',
+ ref=revision,
+ set_got_revision=True)
+ source_path = api.path['start_dir'].join('dart_ci')
+ # Get the Dart SDK to be used to compile the scripts.
+ dart = api.cipd.ensure_tool('dart/dart-sdk/${platform}',
+ 'zysAwdt80NCILRb51ynRCEMypab_UaPrqMsHaEQkpY4C',
+ 'dart-sdk/bin/dart')
+ # Compile scripts.
+ output_root = api.path.mkdtemp()
+ output_file = output_root.join('update_results_database')
+ input_file = source_path.join('builder', 'bin',
+ 'update_results_database.dart')
+ pub_cache = api.path.mkdtemp()
+ with api.context(
+ cwd=source_path.join('builder'), env={'PUB_CACHE': pub_cache}):
+ api.step('pub get', [dart, 'pub', 'get'])
+ api.step('compile script',
+ [dart, 'compile', 'exe', input_file, '-o', output_file])
+ # Create CIPD package from scripts.
+ package_name = 'dart/ci/builder_scripts/${platform}'
+ pkg = api.cipd.PackageDefinition(package_name, output_root, 'copy')
+ pkg.add_file(output_file)
+ api.cipd.create_from_pkg(pkg, tags={'git_revision': got_revision})
+
+
+def GenTests(api):
+ yield api.test(
+ 'basic',
+ api.buildbucket.ci_build(
+ builder='dart_ci_scripts',
+ git_repo='https://dart.googlesource.com/sdk',
+ project='dart'),
+ )
+
+ yield api.test(
+ 'basic-with-revision',
+ api.buildbucket.ci_build(
+ builder='dart_ci_scripts',
+ revision='refs/changes/20/203220/9',
+ git_repo='https://dart.googlesource.com/sdk',
+ project='dart'),
+ )
