recipes/release/debian.py

# Copyright 2023 The Dart Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

import json
import re

from recipe_engine.post_process import (MustRun, DoesNotRunRE)
from PB.recipes.dart.release.debian import Debian

DEPS = [
    'dart',
    'depot_tools/bot_update',
    'depot_tools/depot_tools',
    'depot_tools/gclient',
    'depot_tools/git',
    'depot_tools/gsutil',
    'recipe_engine/context',
    'recipe_engine/buildbucket',
    'recipe_engine/bcid_reporter',
    'recipe_engine/file',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/runtime',
    'recipe_engine/step',
]

PYTHON_VERSION_COMPATIBILITY = 'PY3'

PROPERTIES = Debian


def _build(api, properties):
  with api.depot_tools.on_path():
    api.step('build', ['tools/linux_dist_support/run_debian_build.sh'])


def _upload_build(api, properties, sdk_dir, bucket, topdir, subdir, version):
  directory = sdk_dir.join('out').join('src_and_installation')
  test_data = [
      f'dart-{version}.tar.gz',
      f'dart_{version}-1.debian.tar.xz',
      f'dart_{version}-1.dsc',
      f'dart_{version}-1_amd64.deb',
      f'dart_{version}.orig.tar.gz',
  ]
  files = api.file.listdir(
      'list debian packages', directory, test_data=test_data)
  for file in files:
    name = str(file)[len(str(directory)) + 1:]
    if name.startswith(f'dart-{version}') or name.startswith(f'dart_{version}'):
      if name == f'dart-{version}.tar.gz':
        object_path = f'{topdir}/{subdir}/src/{name}'
      else:
        object_path = f'{topdir}/{subdir}/linux_packages/{name}'
      api.gsutil.upload(
          file,
          bucket,
          object_path,
          name=f'upload {name} ({subdir})',
          dry_run=api.runtime.is_experimental)
      if not properties.disable_bcid and not api.runtime.is_experimental:
        sha256 = api.file.file_hash(file)
        api.bcid_reporter.report_gcs(sha256, f'gs://{bucket}/{object_path}')


def _upload(api, properties, commit, sdk_dir, version, version_string):
  is_new_version = version.channel == 'main' or api.git(
      'diff',
      '--name-only',
      'HEAD~1',
      'tools/VERSION',
      name='check for updated tools/VERSION',
      stdout=api.raw_io.output_text(add_output_log=True)).stdout.rstrip() != ''
  bucket = 'dart-archive'
  topdir = f'channels/{version.channel}/raw'
  if version.channel != 'main':
    subdirs = [f'hash/{commit}']
    if is_new_version:
      subdirs.append('latest')
    for subdir in subdirs:
      _upload_build(api, properties, sdk_dir, bucket, topdir, subdir,
                    version_string)


def _report_stage(api, properties, stage):
  if not properties.disable_bcid:
    api.bcid_reporter.report_stage(stage)


def RunSteps(api, properties):
  _report_stage(api, properties, 'start')
  # Fetch the Dart SDK source code while Snoopy keeps track of dependencies.
  _report_stage(api, properties, 'fetch')
  api.gclient.set_config('dart')
  api.bot_update.ensure_checkout(timeout=25 * 60)  # 25 minutes
  api.gclient.runhooks()
  sdk_dir = api.path['checkout']
  with api.context(cwd=sdk_dir):
    # Build the debian packages.
    _report_stage(api, properties, 'compile')
    commit = api.buildbucket.gitiles_commit.id
    version_path = sdk_dir.join('tools').join('VERSION')
    version_contents = api.file.read_text('read tools/VERSION', version_path)
    version = api.dart.Version(text=version_contents, git_revision=commit)
    version_string = str(version)
    _build(api, properties)
    # Upload the built debian packages to cloud storage.
    _report_stage(api, properties, 'upload')
    builder_name = api.buildbucket.builder_name
    is_try_job = builder_name.endswith('-try')
    # Simulate uploads in experimental mode.
    if api.runtime.is_experimental or not is_try_job:
      _upload(api, properties, commit, sdk_dir, version, version_string)
    _report_stage(api, properties, 'upload-complete')


def GenTests(api):
  yield api.test(
      'debian-x64-stable',
      api.buildbucket.ci_build(
          project='dart-internal',
          builder='debian-x64-stable',
          git_repo='https://dart.googlesource.com/sdk',
          git_ref='refs/heads/stable',
          revision='18df0aae67183931b1b6d5d12cc04d89f7137919'),
      api.properties(Debian()),
      api.step_data(
          'read tools/VERSION',
          api.file.read_text('''
CHANNEL stable
MAJOR 2
MINOR 17
PATCH 3
PRERELEASE 0
PRERELEASE_PATCH 0
''')),
      api.step_data(
          'check for updated tools/VERSION',
          stdout=api.raw_io.output_text('tools/VERSION\n')),
      api.post_process(
          MustRun,
          'gsutil upload dart_2.17.3-1_amd64.deb (hash/18df0aae67183931b1b6d5d12cc04d89f7137919)'
      ),
      api.post_process(MustRun, 'snoop: report_stage (5)'),
  )
  yield api.test(
      'debian-x64-main',
      api.buildbucket.ci_build(
          project='dart-internal',
          builder='debian-x64-main',
          git_repo='https://dart.googlesource.com/sdk',
          git_ref='refs/heads/main',
          revision='18df0aae67183931b1b6d5d12cc04d89f7137919'),
      api.properties(Debian()),
      api.step_data(
          'read tools/VERSION',
          api.file.read_text('''
CHANNEL main
MAJOR 2
MINOR 17
PATCH 0
PRERELEASE 0
PRERELEASE_PATCH 0
''')),
      api.post_process(DoesNotRunRE, 'gsutil upload.*'),
      api.post_process(MustRun, 'snoop: report_stage (5)'),
  )