recipes/release/dartdoc.py

# Copyright 2022 The Dart Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

from PB.recipes.dart.release.release import Release

API_DART_DEV_RELOAD_URL = \
'https://api.dart.dev/docs/releases/latest/?force_reload=true'

BASE_URL = 'gs://dart-api-documentation'
LEGACY_BASE_URL = 'gs://dartlang-api-docs/gen-dartdocs'

DEPS = [
    'dart',
    'depot_tools/gsutil',
    'recipe_engine/archive',
    'recipe_engine/buildbucket',
    'recipe_engine/file',
    'recipe_engine/path',
    'recipe_engine/properties',
    'recipe_engine/raw_io',
    'recipe_engine/runtime',
    'recipe_engine/step',
    'recipe_engine/url',
]

PYTHON_VERSION_COMPATIBILITY = 'PY3'

PROPERTIES = Release


def _exists(api, gs_path, name):
  return api.gsutil(['ls', gs_path], name=name,
                    ok_ret='any').exc_result.retcode == 0


def _gsutil(api, args, name):
  api.gsutil(args, name=name, dry_run=api.runtime.is_experimental)


def RunSteps(api, properties):
  assert not api.buildbucket.builder_name.endswith(
      '-try'), 'tryjob is not supported'
  version_string = properties.version
  assert version_string, 'recipes require a version property'
  version = api.dart.Version(version=version_string)
  channel = version.fields['CHANNEL']
  directory = api.path['cleanup']

  # Download the dartdocs and verify the provenance.
  dartdocs_zip = directory.join('dartdocs-gen-api.zip')
  api.dart.download_and_verify(
      'dartdocs-gen-api.zip', 'dart-archive',
      f'channels/{channel}/release/{version}/api-docs/dartdocs-gen-api.zip',
      dartdocs_zip, 'misc_software://dart/apidocs')
  api.archive.extract('extract dartdocs-gen-api.zip', dartdocs_zip, directory)
  dartdocs_path = directory.join('gen-dartdocs')

  # Publish the dartdocs for the new version.
  _publish_dartdocs(api, version, channel, dartdocs_path, LEGACY_BASE_URL)
  _publish_dartdocs(api, version, channel, dartdocs_path, BASE_URL)

  # Update latest, for stable only if there isn't a newer version.
  if channel != 'stable' or _is_latest_stable(api, version):
    _publish_dartdocs(api, 'latest', channel, dartdocs_path, BASE_URL)

  # Update latest.txt if the new version is newer.
  latest_path = directory.join('latest.txt')
  latest_url = f'gs://dartlang-api-docs/channels/{channel}/latest.txt'
  old_version = None
  if _exists(api, latest_url, f'check for {channel} dartdoc latest.txt'):
    api.gsutil(['cp', latest_url, latest_path], 'download latest.txt')
    old_version_string = api.file.read_text('read latest.txt', latest_path)
    old_version = api.dart.Version(version=old_version_string)
  if not old_version or old_version <= version:
    api.file.write_text('write dartdoc latest.txt', latest_path, str(version))
    _gsutil(
        api, [
            'cp', latest_path,
            f'gs://dartlang-api-docs/channels/{channel}/latest.txt'
        ],
        name='upload dartdocs latest.txt')

  # Trigger a reload to the latest version on the api.dart.dev documentation,
  try:
    api.url.get_text(API_DART_DEV_RELOAD_URL, step_name='reload api.dart.dev')
  except api.step.StepFailure:
    pass


def _publish_dartdocs(api, version, channel, dartdocs_path, base):
  dartdocs_gs_dir = f'{base}/{channel}/{version}'
  _gsutil(api, ['-m', 'cp', '-r', dartdocs_path, dartdocs_gs_dir],
          f'publish dartdocs to {dartdocs_gs_dir}')


def _is_latest_stable(api, version):
  base = f'{BASE_URL}/stable/'
  stable_versions_result = api.gsutil.list(
      base,
      name='list stable versions',
      ok_ret='any',
      stdout=api.raw_io.output_text(add_output_log=True)).stdout
  if not stable_versions_result:
    return True

  # gs://path/<version>/ -> <version>
  stable_versions = map(lambda line: line[len(base):-1],
                        stable_versions_result.splitlines())

  for other_version in stable_versions:
    if other_version == 'latest':
      continue
    if version <= api.dart.Version(version=other_version):
      return False
  return True


def GenTests(api):
  verified = '{"allowed": true,"verificationSummary":"{\\"payload\\": \\"\\"}"}'
  yield api.test(
      'dev',
      api.properties(Release(version='2.17.0-99.0.dev')),
      api.step_data(
          'verify dartdocs-gen-api.zip provenance',
          stdout=api.raw_io.output_text(verified)),
      api.step_data('read latest.txt', api.file.read_text('2.17.0-98.0.dev')),
      api.url.text('reload api.dart.dev', '<html>api.dart.dev</html>'),
  )
  yield api.test(
      'beta',
      api.properties(Release(version='2.17.0-69.1.beta')),
      api.step_data(
          'verify dartdocs-gen-api.zip provenance',
          stdout=api.raw_io.output_text(verified)),
      api.step_data('read latest.txt', api.file.read_text('2.17.0-69.1.beta')),
      api.url.error(
          'reload api.dart.dev', 500, body='500 Internal Server Error'),
  )
  yield api.test(
      'stable',
      api.properties(Release(version='3.2.0')),
      api.step_data(
          'verify dartdocs-gen-api.zip provenance',
          stdout=api.raw_io.output_text(verified)),
      api.step_data('read latest.txt', api.file.read_text('3.1.0')),
      api.step_data(
          'gsutil list stable versions',
          stdout=api.raw_io.output_text(
              '''gs://dart-api-documentation/stable/2.17.2/
gs://dart-api-documentation/stable/3.1.0/
gs://dart-api-documentation/stable/2.17.0/
gs://dart-api-documentation/stable/1.0.5/
gs://dart-api-documentation/stable/latest/
gs://dart-api-documentation/stable/2.0.0/
''')),
      api.url.text('reload api.dart.dev', '<html>api.dart.dev</html>'),
  )
  yield api.test(
      'stable-not-latest',
      api.properties(Release(version='2.17.0')),
      api.step_data(
          'verify dartdocs-gen-api.zip provenance',
          stdout=api.raw_io.output_text(verified)),
      api.step_data('read latest.txt', api.file.read_text('3.1.0')),
      api.step_data(
          'gsutil list stable versions',
          stdout=api.raw_io.output_text(
              '''gs://dart-api-documentation/stable/2.17.2/
gs://dart-api-documentation/stable/3.1.0/
gs://dart-api-documentation/stable/2.17.0/
gs://dart-api-documentation/stable/1.0.5/
latest
gs://dart-api-documentation/stable/2.0.0/
''')),
      api.url.text('reload api.dart.dev', '<html>api.dart.dev</html>'),
  )
  yield api.test(
      'dry',
      api.runtime(is_experimental=True),
      api.properties(Release(version='2.16.0')),
      api.step_data(
          'verify dartdocs-gen-api.zip provenance',
          stdout=api.raw_io.output_text(verified)),
      api.step_data('read latest.txt', api.file.read_text('2.15.0')),
      api.url.text('reload api.dart.dev', '<html>api.dart.dev</html>'),
  )