lib/src/source/hosted.dart

// Copyright (c) 2012, the Dart project authors.  Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.

import 'dart:async';
import 'dart:convert';
import 'dart:io' as io;
import 'dart:io';
import 'dart:typed_data';

import 'package:collection/collection.dart' show IterableExtension;
import 'package:crypto/crypto.dart';
import 'package:http/http.dart' as http;
import 'package:meta/meta.dart';
import 'package:pool/pool.dart';
import 'package:pub_semver/pub_semver.dart';
import 'package:stack_trace/stack_trace.dart';

import '../authentication/client.dart';
import '../crc32c.dart';
import '../exceptions.dart';
import '../http.dart';
import '../io.dart';
import '../language_version.dart';
import '../log.dart' as log;
import '../package.dart';
import '../package_name.dart';
import '../path.dart';
import '../platform_info.dart';
import '../pubspec.dart';
import '../rate_limited_scheduler.dart';
import '../source.dart';
import '../system_cache.dart';
import '../utils.dart';
import 'cached.dart';
import 'root.dart';

const contentHashesDocumentationUrl = 'https://dart.dev/go/content-hashes';

const bool _isDart2Wasm = bool.fromEnvironment('dart.tool.dart2wasm');

/// Validates and normalizes a [hostedUrl] which is pointing to a pub server.
///
/// A [hostedUrl] is a URL pointing to a _hosted pub server_ as defined by the
/// [repository-spec-v2][1]. The default value is `pub.dev`, and can be
/// overwritten using `PUB_HOSTED_URL`. It can also specified for individual
/// hosted-dependencies in `pubspec.yaml`, and for the root package using the
/// `publish_to` key.
///
/// The [hostedUrl] is always normalized to a [Uri] with path that ends in slash
/// unless the path is merely `/`, in which case we normalize to the bare
/// domain.
///
/// We change `https://pub.dartlang.org` to `https://pub.dev`, this  maintains
/// backwards compatibility with `pubspec.lock`-files which contain
/// `https://pub.dartlang.org`.
///
/// Throws [FormatException] if there is anything wrong with [hostedUrl].
///
/// [1]: ../../../doc/repository-spec-v2.md
Uri validateAndNormalizeHostedUrl(String hostedUrl) {
  Uri u;
  try {
    u = Uri.parse(hostedUrl);
  } on FormatException catch (e) {
    throw FormatException('invalid url: ${e.message}', e.source, e.offset);
  }
  if (!u.hasScheme || (u.scheme != 'http' && u.scheme != 'https')) {
    throw FormatException('url scheme must be https:// or http://', hostedUrl);
  }
  if (!u.hasAuthority || u.host == '') {
    throw FormatException('url must have a hostname', hostedUrl);
  }
  if (u.userInfo != '') {
    throw FormatException('user-info is not supported in url', hostedUrl);
  }
  if (u.hasQuery) {
    throw FormatException('querystring is not supported in url', hostedUrl);
  }
  if (u.hasFragment) {
    throw FormatException('fragment is not supported in url', hostedUrl);
  }
  u = u.normalizePath();
  // If we have a path of only `/`
  if (u.path == '/') {
    u = u.replace(path: '');
  }
  // If there is a path, and it doesn't end in a slash we normalize to slash
  if (u.path.isNotEmpty && !u.path.endsWith('/')) {
    u = u.replace(path: '${u.path}/');
  }
  // pub.dev and pub.dartlang.org are identical.
  //
  // We rewrite here to avoid caching both, and to avoid having different
  // credentials for these two.
  //
  // Changing this to pub.dev raises the following concerns:
  //
  //  1. It would blow through users caches.
  //  2. It would cause conflicts for users checking pubspec.lock into git, if
  //     using different versions of the dart-sdk / pub client.
  //  3. It might cause other problems (investigation needed) for pubspec.lock
  //     across different versions of the dart-sdk / pub client.
  //  4. It would expand the API surface we're committed to supporting
  //     long-term.
  //
  // Clearly, a bit of investigation is necessary before we update this to
  // pub.dev, it might be attractive to do next time we change the server API.
  if (u == Uri.parse('https://pub.dartlang.org')) {
    log.fine('Using https://pub.dev instead of https://pub.dartlang.org.');
    u = Uri.parse('https://pub.dev');
  }
  if (runningFromTest &&
      u == Uri.parse('https://pub.dev') &&
      platform.environment.containsKey('_PUB_TEST_DEFAULT_HOSTED_URL')) {
    u = Uri.parse(platform.environment['_PUB_TEST_DEFAULT_HOSTED_URL']!);
  }
  return u;
}

/// A package source that gets packages from a package hosting site that uses
/// the same API as pub.dev.
class HostedSource extends CachedSource {
  static HostedSource instance = HostedSource._();

  HostedSource._();

  @override
  final name = 'hosted';

  static String pubDevUrl = 'https://pub.dev';
  static String pubDartlangUrl = 'https://pub.dartlang.org';

  static bool isPubDevUrl(String url) {
    final parsedUrl = Uri.parse(url);
    if (parsedUrl.scheme != 'http' && parsedUrl.scheme != 'https') {
      // A non http(s) url is not pub.dev.
      return false;
    }
    if (parsedUrl.host.isEmpty) {
      // The empty host is not pub.dev.
      return false;
    }
    final origin = parsedUrl.origin;
    // Allow the defaultHostedUrl to be overriden when running from tests
    if (runningFromTest &&
        platform.environment['_PUB_TEST_DEFAULT_HOSTED_URL'] != null) {
      return origin == platform.environment['_PUB_TEST_DEFAULT_HOSTED_URL'];
    }
    return origin == pubDevUrl || origin == pubDartlangUrl;
  }

  static bool isFromPubDev(PackageId id) {
    final description = id.description.description;
    return description is HostedDescription && isPubDevUrl(description.url);
  }

  /// Gets the default URL for the package server for hosted dependencies.
  late final String defaultUrl = () {
    // Changing this to pub.dev raises the following concerns:
    //
    //  1. It would blow through users caches.
    //  2. It would cause conflicts for users checking pubspec.lock into git, if
    //     using different versions of the dart-sdk / pub client.
    //  3. It might cause other problems (investigation needed) for pubspec.lock
    //     across different versions of the dart-sdk / pub client.
    //  4. It would expand the API surface we're committed to supporting
    //     long-term.
    //
    // Clearly, a bit of investigation is necessary before we update this to
    // pub.dev, it might be attractive to do next time we change the server API.
    try {
      var defaultHostedUrl = 'https://pub.dev';
      // Allow the defaultHostedUrl to be overriden when running from tests
      if (runningFromTest) {
        defaultHostedUrl =
            platform.environment['_PUB_TEST_DEFAULT_HOSTED_URL'] ??
            defaultHostedUrl;
      }
      return validateAndNormalizeHostedUrl(
        platform.environment['PUB_HOSTED_URL'] ?? defaultHostedUrl,
      ).toString();
    } on FormatException catch (e) {
      throw ConfigException(
        'Invalid `PUB_HOSTED_URL="${e.source}"`: ${e.message}',
      );
    }
  }();

  /// Returns a reference to a hosted package named [name].
  ///
  /// If [url] is passed, it's the URL of the pub server from which the package
  /// should be downloaded. [url] will be normalized and validated using
  /// [validateAndNormalizeHostedUrl]. This can throw a [FormatException].
  PackageRef refFor(String name, {String? url}) {
    final d = HostedDescription(name, url ?? defaultUrl);
    return PackageRef(name, d);
  }

  /// Ensures that [description] is a valid hosted package description.
  ///
  /// Simple hosted dependencies only consist of a plain string, which is
  /// resolved against the default host. In this case, [description] will be
  /// null.
  ///
  /// Hosted dependencies may also specify a custom host from which the package
  /// is fetched. There are two syntactic forms of those dependencies:
  ///
  ///  1. With an url and an optional name in a map: `hosted: {url: <url>}`
  ///  2. With a direct url: `hosted: <url>`
  @override
  PackageRef parseRef(
    String name,
    Object? description, {
    required ResolvedDescription containingDescription,
    required LanguageVersion languageVersion,
  }) {
    return PackageRef(
      name,
      _parseDescription(name, description, languageVersion),
    );
  }

  @override
  PackageId parseId(
    String name,
    Version version,
    Object? description, {
    String? containingDir,
  }) {
    // Old pub versions only wrote `description: <pkg>` into the lock file.
    if (description is String) {
      if (description != name) {
        throw const FormatException(
          'The description should be the same as the name',
        );
      }
      return PackageId(
        name,
        version,
        ResolvedHostedDescription(
          HostedDescription(name, defaultUrl),
          sha256: null,
        ),
      );
    }
    if (description is! Map) {
      throw const FormatException(
        'The description should be a string or a map.',
      );
    }
    final url = description['url'];
    if (url is! String) {
      throw const FormatException('The url should be a string.');
    }
    final sha256 = description['sha256'];
    if (sha256 != null && sha256 is! String) {
      throw const FormatException('The sha256 should be a string.');
    }
    final foundName = description['name'];
    if (foundName is! String) {
      throw const FormatException('The name should be a string.');
    }
    if (foundName != name) {
      throw FormatException('The name should be $name');
    }
    return PackageId(
      name,
      version,
      ResolvedHostedDescription(
        HostedDescription(name, url),
        sha256: _parseContentHash(sha256 as String?),
      ),
    );
  }

  /// Decodes a sha256 hash from a lock-file or package-listing.
  /// It is expected to be a hex-encoded String of length 64.
  ///
  /// Throws a [FormatException] if the string cannot be decoded.
  Uint8List? _parseContentHash(String? encoded) {
    if (encoded == null) return null;
    if (encoded.length != 64) {
      throw const FormatException('Content-hash has incorrect length');
    }
    try {
      return hexDecode(encoded);
    } on FormatException catch (e) {
      return throw FormatException(
        'Badly formatted content-hash: ${e.message}',
      );
    }
  }

  /// Parses the description for a package.
  ///
  /// If the package parses correctly, this returns a (name, url) pair. If not,
  /// this throws a descriptive FormatException.
  HostedDescription _parseDescription(
    String packageName,
    Object? description,
    LanguageVersion languageVersion,
  ) {
    if (description == null) {
      // Simple dependency without a `hosted` block, use the default server.
      return HostedDescription(packageName, defaultUrl);
    }

    final canUseShorthandSyntax = languageVersion.supportsShorterHostedSyntax;

    if (description is String) {
      // Old versions of pub (pre Dart 2.15) interpret `hosted: foo` as
      // `hosted: {name: foo, url: <default>}`.
      // For later versions, we treat it as `hosted: {name: <inferred>,
      // url: foo}` if a user opts in by raising their min SDK environment.
      //
      // Since the old behavior is very rarely used and we want to show a
      // helpful error message if the new syntax is used without raising the SDK
      // environment, we throw an error if something that looks like a URI is
      // used as a package name.
      if (canUseShorthandSyntax) {
        return HostedDescription(packageName, description);
      } else {
        if (_looksLikePackageName.hasMatch(description)) {
          // Valid use of `hosted: package` dependency with an old SDK
          // environment.
          return HostedDescription(description, defaultUrl);
        } else {
          const shorterSyntaxVersion =
              LanguageVersion.firstVersionWithShorterHostedSyntax;
          throw FormatException(
            'Using `hosted: <url>` is only supported with a minimum SDK '
            'constraint of $shorterSyntaxVersion.',
          );
        }
      }
    }

    if (description is! Map) {
      throw const FormatException(
        'The description must be a package name or map.',
      );
    }

    var name = description['name'];
    if (canUseShorthandSyntax) name ??= packageName;

    if (name is! String) {
      const shorterSyntaxVersion =
          LanguageVersion.firstVersionWithShorterHostedSyntax;
      throw FormatException(
        "The 'name' key must have a string value without "
        'a minimum Dart SDK constraint of $shorterSyntaxVersion.0 or higher.',
      );
    }

    final u = description['url'];
    if (u != null && u is! String) {
      throw const FormatException("The 'url' key must be a string value.");
    }
    final url = u ?? defaultUrl;

    if (languageVersion.forbidsUnknownDescriptionKeys) {
      for (final key in description.keys) {
        if (!['url', 'name'].contains(key)) {
          throw FormatException('Unknown key "$key" in description.');
        }
      }
    }
    return HostedDescription(name, url as String);
  }

  static final RegExp _looksLikePackageName = RegExp(
    r'^[a-zA-Z_]+[a-zA-Z0-9_]*$',
  );

  List<HostedVersionInfo> _versionInfoFromPackageListing(
    Map body,
    PackageRef ref,
    Uri location,
    SystemCache cache,
  ) {
    final description = ref.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    final versions = body['versions'];
    if (versions is! List) {
      throw const FormatException('versions must be a list');
    }
    return versions.map((map) {
      if (map is! Map) {
        throw const FormatException('versions list element must be a map');
      }
      final pubspecData = map['pubspec'];
      if (pubspecData is! Map) {
        throw const FormatException('pubspec must be a map');
      }

      final archiveSha256 = map['archive_sha256'];
      if (archiveSha256 != null && archiveSha256 is! String) {
        throw const FormatException('archive_sha256 must be a String');
      }
      final parsedContentHash = _parseContentHash(archiveSha256 as String?);
      final pubspec = Pubspec.fromMap(
        pubspecData,
        cache.sources,
        expectedName: ref.name,
        location: location,
        containingDescription: ResolvedHostedDescription(
          description,
          sha256: parsedContentHash,
        ),
      );
      final archiveUrl = map['archive_url'];
      if (archiveUrl is! String) {
        throw const FormatException('archive_url must be a String');
      }
      final isDiscontinued = body['isDiscontinued'] ?? false;
      if (isDiscontinued is! bool) {
        throw const FormatException('isDiscontinued must be a bool');
      }
      final replacedBy = body['replacedBy'];
      if (replacedBy is! String?) {
        throw const FormatException('replacedBy must be a String');
      }
      final retracted = map['retracted'] ?? false;
      if (retracted is! bool) {
        throw const FormatException('retracted must be a bool');
      }
      DateTime? advisoriesDate;
      final advisoriesUpdated = body['advisoriesUpdated'];
      if (advisoriesUpdated != null) {
        if (advisoriesUpdated is! String) {
          throw const FormatException('advisoriesUpdated must be a String');
        }
        advisoriesDate = DateTime.parse(advisoriesUpdated);
      }
      final status = PackageStatus(
        isDiscontinued: isDiscontinued,
        discontinuedReplacedBy:
            replacedBy == null ? null : sanitizeForTerminal(replacedBy),
        isRetracted: retracted,
        advisoriesUpdated: advisoriesDate,
      );
      return HostedVersionInfo(
        pubspec.version,
        pubspec,
        Uri.parse(archiveUrl),
        status,
        parsedContentHash,
      );
    }).toList();
  }

  Future<List<HostedVersionInfo>> _fetchVersionsNoPrefetching(
    PackageRef ref,
    SystemCache cache,
  ) async {
    final description = ref.description;

    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    final packageName = description.packageName;
    final hostedUrl = description.url;
    final url = _listVersionsUrl(ref);
    log.io('Get versions from $url.');

    // Only attach If-None-Match when running on native platforms (dart:io).
    // In browser environments (e.g. web tests), If-None-Match triggers a CORS
    // preflight OPTIONS request which package repositories may not support.
    final (cachedEtag, cachedDoc) =
        const bool.fromEnvironment('dart.library.io')
            ? _cachedVersionListingDoc(ref, cache)
            : (null, null);
    final String? bodyText;
    final dynamic body;
    final List<HostedVersionInfo> result;
    try {
      // TODO(sigurdm): Implement cancellation of requests. This probably
      // requires resolution of: https://github.com/dart-lang/http/issues/424.
      final (responseBody, responseEtag, is304) = await withAuthenticatedClient(
        cache,
        Uri.parse(hostedUrl),
        (client) async {
          return await retryForHttp(
            'fetching versions for "$packageName" from "$url"',
            () async {
              final request = http.Request('GET', url);
              request.attachPubApiHeaders();
              if (cachedEtag != null) {
                request.headers[HttpHeaders.ifNoneMatchHeader] = cachedEtag;
              }
              final response = await client.fetch(request);
              if (response.statusCode == HttpStatus.notModified) {
                return (null, null, true);
              }
              return (
                response.body,
                response.headers[HttpHeaders.etagHeader],
                false,
              );
            },
          );
        },
      );

      if (is304) {
        if (cachedDoc != null) {
          try {
            final res = _versionInfoFromPackageListing(
              cachedDoc,
              ref,
              Uri.file(_versionListingCachePath(ref, cache)),
              cache,
            );
            cache.hostedCache._responseCache[ref] = (DateTime.now(), res);
            log.io('Version listing for "$packageName" is not modified (304).');
            return res;
          } on FormatException {
            // Cached document was invalid, fall through to refetch.
          }
        }
        // If the cache was missing/corrupted despite having an ETag, refetch
        // unconditionally.
        tryDeleteEntry(_versionListingCachePath(ref, cache));
        return await _fetchVersionsNoPrefetching(ref, cache);
      }

      bodyText = responseBody!;
      final decoded = jsonDecode(bodyText);
      if (decoded is! Map<String, dynamic>) {
        throw const FormatException('version listing must be a mapping');
      }
      body = decoded;
      result = _versionInfoFromPackageListing(
        body as Map<String, dynamic>,
        ref,
        url,
        cache,
      );

      // Cache the response on disk.
      // Don't cache overly big responses.
      if (bodyText.length < 1000 * 1024) {
        await _cacheVersionListingResponse(
          body,
          ref,
          cache,
          etag: responseEtag,
        );
      }
      return result;
    } on Exception catch (error, stackTrace) {
      _throwFriendlyError(error, stackTrace, packageName, hostedUrl);
    }
  }

  Future<List<HostedVersionInfo>> fetchVersions(
    HostedRefAndCache refAndCache,
  ) async {
    final ref = refAndCache.ref;
    final description = ref.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    final preschedule =
        Zone.current[_prefetchingKey] as void Function(HostedRefAndCache)?;

    /// Prefetch the dependencies of the latest version, we are likely to need
    /// them later.
    void prescheduleDependenciesOfLatest(
      List<HostedVersionInfo>? listing,
      SystemCache cache,
    ) {
      if (listing == null || listing.isEmpty) return;
      var latestVersion = listing.first;
      for (final e in listing.skip(1)) {
        if (e.version > latestVersion.version) {
          latestVersion = e;
        }
      }
      final dependencies = latestVersion.pubspec.dependencies.values;
      unawaited(() async {
        for (final packageRange in dependencies) {
          if (packageRange.source is HostedSource) {
            preschedule!(HostedRefAndCache(packageRange.toRef(), cache));
          }
        }
      }());
    }

    final cache = refAndCache.cache;
    if (preschedule != null) {
      /// If we have a cached response - preschedule dependencies of that.
      prescheduleDependenciesOfLatest(
        await _cachedVersionListingResponse(ref, cache),
        cache,
      );
    }
    final result = await _fetchVersionsNoPrefetching(ref, cache);

    if (preschedule != null) {
      // Preschedule the dependencies from the actual response.
      // This might overlap with those from the cached response. But the
      // scheduler ensures each listing will be fetched at most once.
      prescheduleDependenciesOfLatest(result, cache);
    }
    return result;
  }

  Future<List<Advisory>?> _fetchAdvisories(
    PackageRef ref,
    SystemCache cache,
  ) async {
    if (cache.isOffline) {
      // Don't attempt to fetch advisories in `--offline` mode.
      return null;
    }
    final description = ref.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }

    final packageName = description.packageName;
    final hostedUrl = description.url;

    final url = _listAdvisoriesUrl(ref);
    log.io('Fetching security advisories from $url.');

    final String bodyText;
    final Map<String, dynamic> body;
    final List<Advisory>? result;
    try {
      bodyText = await cache.hostedCache.pool.withResource(() async {
        return await withAuthenticatedClient(cache, Uri.parse(hostedUrl), (
          client,
        ) async {
          return await retryForHttp(
            'fetching advisories for "$packageName" from "$url"',
            () async {
              final request = http.Request('GET', url);
              request.attachPubApiHeaders();
              final response = await client.fetch(request);
              return response.body;
            },
          );
        });
      });
      final decoded = jsonDecode(bodyText);
      if (decoded is! Map<String, dynamic>) {
        throw const FormatException('security advisories must be a mapping');
      }
      body = decoded;
      result = _extractAdvisoryDetailsForPackage(decoded, ref.name);
    } on FormatException catch (error) {
      log.warning(
        'Failed to decode advisories for $packageName from $hostedUrl: '
        '${getErrorMessage(error)}',
      );
      return null;
    } on PubHttpResponseException catch (error, stackTrace) {
      if (isPubDevUrl(hostedUrl)) {
        fail(
          'Failed to fetch advisories for "$packageName" from "$hostedUrl".\n',
          error,
          stackTrace,
        );
      } else {
        log.warning(
          'Warning: Unable to fetch advisories for "$packageName" '
          'from "$hostedUrl".\n',
        );
      }
      return null;
    }

    // Cache the response on disk.
    // Don't cache overly big responses.
    if (bodyText.length < 5000 * 1024) {
      await _cacheAdvisoriesResponse(body, ref, cache);
    }
    return result;
  }

  /// Extracts relevant details from [body] about advisories affecting the
  /// package named [packageName]. The details are used when generating output
  /// for runs such as `dart pub upgrade`.
  List<Advisory>? _extractAdvisoryDetailsForPackage(
    Map body,
    String packageName,
  ) {
    final advisoriesList = <Advisory>[];

    final advisories = body['advisories'];
    if (advisories is! List) {
      throw const FormatException('advisories must be a list');
    }

    final advisoriesUpdated = body['advisoriesUpdated'];
    if (advisoriesUpdated is! String) {
      throw const FormatException('advisoriesUpdated must be a String');
    }

    for (final advisory in advisories) {
      if (advisory is! Map) {
        throw const FormatException('advisory must be a map');
      }

      final databaseSpecific = advisory['database_specific'];
      if (databaseSpecific is! Map?) {
        throw const FormatException('database_specific must be a map or null');
      }

      final pubDisplayUrl = databaseSpecific?['pub_display_url'];
      if (pubDisplayUrl is! String?) {
        throw const FormatException('pub_display_url must be a String or null');
      }

      final id = advisory['id'];
      if (id is! String) {
        throw const FormatException('id must be a String');
      }

      final summary = advisory['summary'];
      if (summary is! String) {
        throw const FormatException('summary must be a String');
      }

      final aliasIDs = <String>[];
      final aliases = advisory['aliases'];
      if (aliases is! List) {
        throw const FormatException('aliases must be a list');
      }

      for (final id in aliases) {
        if (id is! String) {
          throw const FormatException('alias IDs must be a string');
        }
        aliasIDs.add(id);
      }

      final affectedPackages = advisory['affected'];
      if (affectedPackages is! List) {
        throw const FormatException('affectedPackages must be a list');
      }

      bool matchesNameAndEcosystem(
        dynamic affectedPackage,
        String name,
        String ecosystem,
      ) {
        if (affectedPackage is! Map) {
          throw const FormatException('`affected` must be a map');
        }
        final package = affectedPackage['package'];
        if (package is! Map) {
          throw const FormatException('`package` must be a map');
        }
        final affectedName = package['name'];
        if (affectedName is! String) {
          throw const FormatException('package name must be a String');
        }
        if (affectedName == name) {
          final affectedEcosystem = package['ecosystem'];
          if (affectedEcosystem is! String) {
            throw const FormatException('ecosystem must be a String');
          }
          return affectedEcosystem.toLowerCase() == ecosystem;
        }
        return false;
      }

      for (final affectedPackage in affectedPackages) {
        if (affectedPackage is! Map) {
          throw const FormatException('`affected` must be a list of maps');
        }
        if (matchesNameAndEcosystem(affectedPackage, packageName, 'pub')) {
          final affectedVersions = <String>{};
          final versions = affectedPackage['versions'];
          if (versions is! List) {
            throw const FormatException('package versions must be a list');
          }

          for (final v in versions) {
            if (v is! String) {
              throw const FormatException(
                'package version elements must be a string',
              );
            }
            affectedVersions.add(v);
          }

          advisoriesList.add(
            Advisory(
              id: id,
              affectedVersions: affectedVersions,
              aliases: aliasIDs,
              summary: summary,
              pubDisplayUrl: pubDisplayUrl,
            ),
          );
        }
      }
    }

    return advisoriesList;
  }

  Future<List<Advisory>?> _getAdvisories(
    PackageId id,
    SystemCache cache,
    Duration? maxAge,
  ) async {
    final advisoriesUpdated =
        (await status(
          id.toRef(),
          id.version,
          cache,
          maxAge: maxAge,
        )).advisoriesUpdated;
    if (advisoriesUpdated == null) return null;

    Future<List<Advisory>?> readAdvisoriesFromCache() async {
      final advisoriesCachePath = _advisoriesCachePath(id.toRef(), cache);
      final stat = io.File(advisoriesCachePath).statSync();

      if (stat.type == io.FileSystemEntityType.file) {
        // To tolerate timezone differences and clock skew between the local
        // machine and the pub.dev server, we add a 24-hour buffer to
        // stat.modified.
        if (advisoriesUpdated.isAfter(
          stat.modified.add(const Duration(hours: 24)),
        )) {
          tryDeleteEntry(advisoriesCachePath);
          return null;
        }

        try {
          final doc = jsonDecode(readTextFile(advisoriesCachePath));
          if (doc is! Map) {
            throw const FormatException('Broken cached advisories response');
          }

          final cachedAdvisoriesUpdated = doc['advisoriesUpdated'];
          if (cachedAdvisoriesUpdated is! String) {
            throw const FormatException('Broken cached advisories response');
          }
          final parsedCacheAdvisoriesUpdated = DateTime.parse(
            cachedAdvisoriesUpdated,
          );
          if (advisoriesUpdated.isAfter(parsedCacheAdvisoriesUpdated)) {
            tryDeleteEntry(advisoriesCachePath);
          } else {
            return _extractAdvisoryDetailsForPackage(doc, id.toRef().name);
          }
        } on io.IOException {
          // Could not read the file. Delete if it exists.
          tryDeleteEntry(advisoriesCachePath);
        } on FormatException catch (e) {
          tryDeleteEntry(advisoriesCachePath);
          log.fine('Failed to read cached advisories: $e');
        }
      }
      return null;
    }

    return await readAdvisoriesFromCache() ??
        await _fetchAdvisories(id.toRef(), cache);
  }

  /// If a cached version listing response for [ref] exists on disk and is less
  /// than [maxAge] old it is parsed and returned.
  ///
  /// Otherwise deletes a cached response if it exists and returns `null`.
  ///
  /// If [maxAge] is not given, we will try to get the cached version no matter
  /// how old it is.
  Future<List<HostedVersionInfo>?> _cachedVersionListingResponse(
    PackageRef ref,
    SystemCache cache, {
    Duration? maxAge,
  }) async {
    final cachedInfo = cache.hostedCache._responseCache[ref];
    if (cachedInfo != null) {
      final (cacheTimestamp, versionInfo) = cachedInfo;
      final cacheAge = DateTime.now().difference(cacheTimestamp);
      if (maxAge == null || maxAge > cacheAge) {
        // The cached value is not too old.
        return versionInfo;
      }
    }
    final cachePath = _versionListingCachePath(ref, cache);
    final stat = io.File(cachePath).statSync();
    final now = DateTime.now();
    if (stat.type == io.FileSystemEntityType.file) {
      if (maxAge == null || now.difference(stat.modified) < maxAge) {
        try {
          final cachedDoc = jsonDecode(readTextFile(cachePath));
          if (cachedDoc is! Map) {
            throw const FormatException(
              'Broken cached version listing response',
            );
          }
          final timestamp = cachedDoc['_fetchedAt'];
          if (timestamp is! String) {
            throw const FormatException(
              'Broken cached version listing response',
            );
          }
          final parsedTimestamp = DateTime.parse(timestamp);
          final cacheAge = DateTime.now().difference(parsedTimestamp);
          if (maxAge != null && cacheAge > maxAge) {
            // Too old according to internal timestamp - delete.
            tryDeleteEntry(cachePath);
          } else {
            final res = _versionInfoFromPackageListing(
              cachedDoc,
              ref,
              Uri.file(cachePath),
              cache,
            );
            cache.hostedCache._responseCache[ref] = (parsedTimestamp, res);
            return res;
          }
        } on io.IOException {
          // Could not read the file. Delete if it exists.
          tryDeleteEntry(cachePath);
        } on FormatException {
          // Decoding error - bad file or bad timestamp. Delete the file.
          tryDeleteEntry(cachePath);
        }
      } else {
        // File too old
        tryDeleteEntry(cachePath);
      }
    }
    return null;
  }

  Future<void> _cacheAdvisoriesResponse(
    Map<String, dynamic> body,
    PackageRef ref,
    SystemCache cache,
  ) async {
    final path = _advisoriesCachePath(ref, cache);
    try {
      ensureDir(p.dirname(path));

      writeTextFile(path, jsonEncode(<String, dynamic>{...body}));
    } on io.IOException catch (e) {
      // Not being able to write this cache is not fatal. Just move on...
      log.fine('Failed writing cache file. $e');
    }
  }

  /// Matches an RFC 9110 compliant entity-tag (e.g. `"abc123"` or `W/"abc123"`).
  static final RegExp _etagRegExp = RegExp(r'^(?:W/)?"[\x21\x23-\x7e]*"$');

  /// Returns true if [etag] is a syntactically valid RFC 9110 entity-tag and
  /// does not exceed reasonable length limits.
  static bool _isValidETag(String? etag) {
    if (etag == null) return false;
    final trimmed = etag.trim();
    if (trimmed.isEmpty || trimmed.length > 512) return false;
    return _etagRegExp.hasMatch(trimmed);
  }

  /// Returns the cached ETag and decoded JSON document for [ref] if one is
  /// stored in `.cache/<pkg>-versions.json`.
  (String? etag, Map<String, dynamic>? doc) _cachedVersionListingDoc(
    PackageRef ref,
    SystemCache cache,
  ) {
    final cachePath = _versionListingCachePath(ref, cache);
    try {
      final cachedDoc = jsonDecode(readTextFile(cachePath));
      if (cachedDoc is Map<String, dynamic>) {
        final etag = cachedDoc['_etag'];
        final validEtag =
            etag is String && _isValidETag(etag) ? etag.trim() : null;
        return (validEtag, cachedDoc);
      }
    } on io.IOException {
      // Ignore read errors (e.g. file does not exist).
    } on FormatException {
      // Corrupted file on disk. Delete it so subsequent requests don't trip.
      tryDeleteEntry(cachePath);
    }
    return (null, null);
  }

  /// Saves the (decoded) response from package-listing of [ref].
  Future<void> _cacheVersionListingResponse(
    Map<String, dynamic> body,
    PackageRef ref,
    SystemCache cache, {
    String? etag,
  }) async {
    final path = _versionListingCachePath(ref, cache);
    final validEtag = _isValidETag(etag) ? etag!.trim() : null;
    try {
      ensureDir(p.dirname(path));
      await writeTextFileAsync(
        path,
        jsonEncode(<String, dynamic>{
          ...body,
          '_fetchedAt': DateTime.now().toIso8601String(),
          if (validEtag != null) '_etag': validEtag,
        }),
      );
      // Delete the entry in the in-memory cache to maintain the invariant that
      // cached information in memory is the same as that on the disk.
      cache.hostedCache._responseCache.remove(ref);
    } on io.IOException catch (e) {
      // Not being able to write this cache is not fatal. Just move on...
      log.fine('Failed writing cache file. $e');
    }
  }

  @override
  Future<PackageStatus> status(
    PackageRef ref,
    Version version,
    SystemCache cache, {
    Duration? maxAge,
  }) async {
    // If we don't have the specific version we return the empty response, since
    // it is more or less harmless..
    //
    // This can happen if the connection is broken, or the server is faulty.
    // We want to avoid a crash
    //
    // TODO(sigurdm): Consider representing the non-existence of the
    // package-version in the return value.
    return (await _versionInfo(ref, version, cache, maxAge: maxAge))?.status ??
        PackageStatus();
  }

  Future<HostedVersionInfo?> _versionInfo(
    PackageRef ref,
    Version version,
    SystemCache cache, {
    Duration? maxAge,
  }) async {
    if (cache.isOffline) {
      // Do we have a cached version response on disk?
      final versionListing = await _cachedVersionListingResponse(ref, cache);

      if (versionListing == null) {
        return null;
      }
      return versionListing.firstWhereOrNull(
        (HostedVersionInfo l) => l.version == version,
      );
    }
    // Did we already get info for this package?
    var versionListing = cache.hostedCache.scheduler.peek(
      HostedRefAndCache(ref, cache),
    );
    if (maxAge != null) {
      // Do we have a cached version response on disk?
      versionListing ??= await _cachedVersionListingResponse(
        ref,
        cache,
        maxAge: maxAge,
      );
    }
    // Otherwise retrieve the info from the host.
    versionListing ??= await cache.hostedCache.scheduler
        .schedule(HostedRefAndCache(ref, cache))
        // Failures retrieving the listing here should just be ignored.
        .catchError(
          (_) async => <HostedVersionInfo>[],
          test: (error) => error is Exception,
        );

    return versionListing.firstWhereOrNull((l) => l.version == version);
  }

  // The path to .cache directory inside the pub cache
  String _cacheDirPath(PackageRef ref, SystemCache cache) {
    final description = ref.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    final dir = _urlToDirectory(description.url);
    // Use a dot-dir because older versions of pub won't choke on that
    // name when iterating the cache (it is not listed by [listDir]).
    return p.join(cache.rootDirForSource(this), dir, _versionListingDirectory);
  }

  static const _versionListingDirectory = '.cache';

  // The path where the response from the package-listing api is cached.
  String _versionListingCachePath(PackageRef ref, SystemCache cache) {
    return p.join(_cacheDirPath(ref, cache), '${ref.name}-versions.json');
  }

  // The path where the response from the advisories api is cached.
  String _advisoriesCachePath(PackageRef ref, SystemCache cache) {
    return p.join(_cacheDirPath(ref, cache), '${ref.name}-advisories.json');
  }

  /// Downloads a list of all versions of a package that are available from the
  /// site.
  @override
  Future<List<PackageId>> doGetVersions(
    PackageRef ref,
    Duration? maxAge,
    SystemCache cache,
  ) async {
    final description = ref.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    if (cache.isOffline) {
      final url = description.url;
      final root = cache.rootDirForSource(HostedSource.instance);
      final dir = p.join(root, _urlToDirectory(url));
      log.io('Finding versions of ${ref.name} in $dir');
      List<PackageId> offlineVersions;
      if (dirExists(dir)) {
        offlineVersions =
            listDir(dir)
                .where(_looksLikePackageDir)
                .map((entry) => _idForBasename(p.basename(entry), url))
                .where(
                  (id) => id.name == ref.name && id.version != Version.none,
                )
                .toList();
      } else {
        offlineVersions = [];
      }

      // If there are no versions in the cache, report a clearer error.
      if (offlineVersions.isEmpty) {
        throw PackageNotFoundException(
          'could not find package ${ref.name} in cache',
          hint: 'Try again without --offline!',
        );
      }

      return offlineVersions;
    }
    var versionListing = cache.hostedCache.scheduler.peek(
      HostedRefAndCache(ref, cache),
    );
    if (maxAge != null) {
      // Do we have a cached version response on disk?
      versionListing ??= await _cachedVersionListingResponse(
        ref,
        cache,
        maxAge: maxAge,
      );
    }
    versionListing ??= await cache.hostedCache.scheduler.schedule(
      HostedRefAndCache(ref, cache),
    );
    return versionListing
        .map(
          (i) => PackageId(
            ref.name,
            i.version,
            ResolvedHostedDescription(
              ref.description as HostedDescription,
              sha256: i.archiveSha256,
            ),
          ),
        )
        .toList();
  }

  @override
  Future<List<Advisory>?>? getAdvisoriesForPackage(
    PackageId id,
    SystemCache cache,
    Duration? maxAge,
  ) {
    return cache.hostedCache._advisoriesCache.putIfAbsent(
      id,
      () => _getAdvisories(id, cache, maxAge),
    );
  }

  @override
  Future<List<Advisory>?>? getAdvisoriesForPackageVersion(
    PackageId id,
    SystemCache cache,
    Duration? maxAge,
  ) async {
    final advisories = await getAdvisoriesForPackage(id, cache, maxAge);
    if (advisories == null) return null;

    return advisories
        .where(
          (advisory) => advisory.affectedVersions.contains(
            id.version.canonicalizedVersion,
          ),
        )
        .toList();
  }

  /// Parses [ref] into its hosted description and package name components.
  (HostedDescription, String) _parseRef(PackageRef ref) {
    final description = ref.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    final package = Uri.encodeComponent(ref.name);
    return (description, package);
  }

  /// Parses [ref] and computes the api url for listing versions of the given
  /// package.
  Uri _listVersionsUrl(PackageRef ref) {
    final (description, package) = _parseRef(ref);
    return Uri.parse(description.url).resolve('api/packages/$package');
  }

  /// Parses [ref] and computes the api url for getting security advisories for
  /// a given package.
  Uri _listAdvisoriesUrl(PackageRef ref) {
    final (description, package) = _parseRef(ref);
    return Uri.parse(
      description.url,
    ).resolve('api/packages/$package/advisories');
  }

  /// Retrieves the pubspec for a specific version of a package that is
  /// available from the site.
  @override
  Future<Pubspec> describeUncached(PackageId id, SystemCache cache) async {
    if (cache.isOffline) {
      throw PackageNotFoundException(
        '${id.name} ${id.version} is not available in cache',
        hint: 'Try again without --offline!',
      );
    }
    final versions = await cache.hostedCache.scheduler.schedule(
      HostedRefAndCache(id.toRef(), cache),
    );
    final url = _listVersionsUrl(id.toRef());
    return versions.firstWhereOrNull((i) => i.version == id.version)?.pubspec ??
        (throw PackageNotFoundException('Could not find package $id at $url'));
  }

  /// Downloads the package identified by [id] to the system cache if needed.
  ///
  /// Validates that the content hash of [id] corresponds to what is already in
  /// cache, if not the file is redownloaded.
  @override
  Future<DownloadPackageResult> downloadToSystemCache(
    PackageId id,
    SystemCache cache,
  ) async {
    var didUpdate = false;
    final packageDir = getDirectoryInCache(id, cache);

    // Use the content-hash from the version-info to compare with what we
    // already downloaded.
    //
    // The content-hash from [id] will be compared with that when the lockfile
    // is written.
    //
    // We allow the version-listing to be a few days outdated in order for `pub
    // get` with an existing working resolution and everything in cache to be
    // fast.
    final versionInfo = await _versionInfo(
      id.toRef(),
      id.version,
      cache,
      maxAge: const Duration(days: 3),
    );

    final expectedContentHash =
        versionInfo?.archiveSha256 ??
        // Handling of legacy server - we use the hash from the id (typically
        // from the lockfile) to compare to the existing download.
        (id.description as ResolvedHostedDescription).sha256;
    Uint8List? contentHash;
    if (!fileExists(hashPath(id, cache))) {
      if (dirExists(packageDir) && !cache.isOffline) {
        log.fine(
          'Cache entry for ${id.name}-${id.version} has no content-hash '
          '- redownloading.',
        );
        deleteEntry(packageDir);
      }
    } else if (expectedContentHash == null) {
      // Can happen with a legacy server combined with a legacy lock file.
      log.fine(
        'Content-hash of ${id.name}-${id.version} not known from resolution.',
      );
    } else {
      final hashFromCache = sha256FromCache(id, cache);
      if (!fixedTimeBytesEquals(hashFromCache, expectedContentHash)) {
        log.warning(
          'Cached version of ${id.name}-${id.version} has wrong hash '
          '- redownloading.',
        );
        if (cache.isOffline) {
          fail('Cannot redownload while offline. Try again without --offline.');
        }
        deleteEntry(packageDir);
      } else {
        contentHash = hashFromCache;
      }
    }
    if (dirExists(packageDir)) {
      contentHash ??= sha256FromCache(id, cache);
    } else {
      didUpdate = true;
      if (cache.isOffline) {
        fail(
          'Missing package ${id.name}-${id.version}. '
          'Try again without --offline.',
        );
      }
      contentHash = await _downloadAtomically(id, packageDir, cache);
    }
    return DownloadPackageResult(
      PackageId(
        id.name,
        id.version,
        (id.description as ResolvedHostedDescription).withSha256(contentHash),
      ),
      didUpdate: didUpdate,
    );
  }

  /// The system cache directory for the hosted source contains subdirectories
  /// for each separate repository URL that's used on the system.
  ///
  /// Each of these subdirectories then contains a subdirectory for each
  /// package downloaded from that site.
  @override
  String getDirectoryInCache(PackageId id, SystemCache cache) {
    final description = id.description.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    final rootDir = cache.rootDirForSource(this);

    final dir = _urlToDirectory(description.url);
    return p.join(rootDir, dir, '${id.name}-${id.version}');
  }

  /// The system cache directory for the hosted source contains subdirectories
  /// for each separate repository URL that's used on the system.
  ///
  /// Parallel to this there is a `hosted-hashes` directory with a stored hash
  /// of all downloaded packages.
  String hashPath(PackageId id, SystemCache cache) {
    final description = id.description.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    final rootDir = cache.rootDir;

    final serverDir = _urlToDirectory(description.url);
    return p.join(
      rootDir,
      'hosted-hashes',
      serverDir,
      '${id.name}-${id.version}.sha256',
    );
  }

  /// Loads the hash at `hashPath(id)`.
  Uint8List? sha256FromCache(PackageId id, SystemCache cache) {
    try {
      return _parseContentHash(readTextFile(hashPath(id, cache)));
    } on io.IOException {
      return null;
    } on FormatException catch (e) {
      log.fine('Bad content-hash in cache: $e, ignoring cache entry');
      return null;
    }
  }

  /// Re-downloads all packages that have been previously downloaded into the
  /// system cache from any server.
  ///
  /// If [packageFilter] is provided, only packages whose names are in the set
  /// will be repaired.
  @override
  Future<Iterable<RepairResult>> repairCachedPackages(
    SystemCache cache, {
    bool Function(String name, Version version)? packageFilter,
  }) async {
    final rootDir = cache.rootDirForSource(this);
    if (!dirExists(rootDir)) return [];

    return (await Future.wait(
      listDir(rootDir).map((serverDir) async {
        final directory = p.basename(serverDir);
        final String url;
        try {
          url = _directoryToUrl(directory);
        } on FormatException {
          log.error('Unable to detect hosted url from directory: $directory');
          // If _directoryToUrl can't interpret a directory name,
          // we just silently ignore it and hope it's because it
          // comes from a newer version of pub.
          //
          // This is most likely because someone manually modified PUB_CACHE.
          return <RepairResult>[];
        }

        final results = <RepairResult>[];
        final packages = <Package>[];
        for (var entry in listDir(serverDir)) {
          final id = _idForBasename(p.basename(entry), url);
          if (packageFilter != null && !packageFilter(id.name, id.version)) {
            continue;
          }

          try {
            final package = Package.load(
              entry,
              loadPubspec: Pubspec.loadRootWithSources(cache.sources),
            );
            packages.add(package);
          } catch (error, stackTrace) {
            log.error('Failed to load package', error, stackTrace);
            results.add(
              RepairResult(id.name, id.version, this, success: false),
            );
            tryDeleteEntry(entry);
          }
        }

        // Delete the cached package listings.
        tryDeleteEntry(p.join(serverDir, _versionListingDirectory));

        packages.sort(Package.orderByNameAndVersion);

        final pool = Pool(10);
        return results..addAll(
          await Future.wait(
            packages.map((package) async {
              return await pool.withResource(() async {
                final id = PackageId(
                  package.name,
                  package.version,
                  ResolvedHostedDescription(
                    HostedDescription._(package.name, url),
                    sha256: null,
                  ),
                );
                try {
                  deleteEntry(package.dir);
                  await _downloadAtomically(id, package.dir, cache);
                  return RepairResult(id.name, id.version, this, success: true);
                } catch (error, stackTrace) {
                  var message =
                      'Failed to repair ${log.bold(package.name)} '
                      '${package.version}';
                  if (url != defaultUrl) message += ' from $url';
                  log.error('$message. Error:\n$error');
                  log.fine(stackTrace.toString());

                  tryDeleteEntry(package.dir);
                  return RepairResult(
                    id.name,
                    id.version,
                    this,
                    success: false,
                  );
                }
              });
            }),
          ),
        );
      }),
    )).expand((x) => x);
  }

  /// Returns the best-guess package ID for [basename], which should be a
  /// subdirectory in a hosted cache.
  PackageId _idForBasename(String basename, String url) {
    final components = split1(basename, '-');
    var version = Version.none;
    if (components.length > 1) {
      try {
        version = Version.parse(components.last);
      } on FormatException {
        // Default to Version.none.
      }
    }
    final name = components.first;
    return PackageId(
      name,
      version,
      ResolvedHostedDescription(HostedDescription(name, url), sha256: null),
    );
  }

  bool _looksLikePackageDir(String path) {
    final components = split1(p.basename(path), '-');
    if (components.length < 2) return false;
    try {
      Version.parse(components.last);
    } on FormatException {
      return false;
    }
    return dirExists(path);
  }

  /// Gets all of the packages that have been downloaded into the system cache
  /// from the default server.
  @override
  List<Package> getCachedPackages(SystemCache cache) {
    final root = cache.rootDirForSource(HostedSource.instance);
    final cacheDir = p.join(
      root,
      _urlToDirectory(HostedSource.instance.defaultUrl),
    );
    if (!dirExists(cacheDir)) return [];

    return listDir(cacheDir)
        .where(_looksLikePackageDir)
        .map((entry) {
          try {
            return Package.load(
              entry,
              loadPubspec: Pubspec.loadRootWithSources(cache.sources),
            );
          } catch (error, stackTrace) {
            log.fine(
              'Failed to load package from $entry:\n'
              '$error\n'
              '${Chain.forTrace(stackTrace)}',
            );
            return null;
          }
        })
        .nonNulls
        .toList();
  }

  Future<void> downloadInto(
    PackageId id,
    String destPath,
    SystemCache cache,
  ) async {
    try {
      // For the functionality of `unpack` it is important that the unpack
      // doesn't go via the cache, as that would not allow unpacking into a
      // directory on a different device than the cache. So we don't use
      // `_downloadAtomically` here.
      await _downloadAndExtract(id, destPath, cache);
    } catch (e) {
      tryDeleteEntry(destPath);
      rethrow;
    }
  }

  /// Downloads package [id] from the archive_url and unpacks it into
  /// [destPath]. The unpack is done to a temporary directory and then moved
  /// into [destPath] atomically.
  ///
  /// If there is no archive_url, try to fetch it from
  /// `$server/packages/$package/versions/$version.tar.gz` where server comes
  /// from `id.description`.
  ///
  /// Returns the content-hash of the downloaded archive.
  Future<Uint8List> _downloadAtomically(
    PackageId id,
    String destPath,
    SystemCache cache,
  ) async {
    final tempDir = cache.createTempDir();
    try {
      final contentHash = await _downloadAndExtract(id, tempDir, cache);
      ensureDir(p.dirname(destPath));
      tryRenameDir(tempDir, destPath);
      return contentHash;
    } catch (e) {
      deleteEntry(tempDir);
      rethrow;
    }
  }

  /// Downloads the archive for [id] and extracts it to [destPath].
  ///
  /// Returns the content-hash of the downloaded archive.
  Future<Uint8List> _downloadAndExtract(
    PackageId id,
    String destPath,
    SystemCache cache,
  ) async {
    final description = id.description.description;
    if (description is! HostedDescription) {
      throw ArgumentError('Wrong source');
    }
    // We never want to use a cached `archive_url`, so we never attempt to load
    // the version listing from cache. Besides in most cases we already have
    // downloaded a fresh copy of the version listing response in the in-memory
    // cache, so looking in the file-system is pointless.
    //
    // We avoid using cached `archive_url` values because the `archive_url` for
    // a custom package server may include a temporary signature in the
    // query-string as is the case with signed S3 URLs. And we wish to allow for
    // such URLs to be used.
    final versions = await cache.hostedCache.scheduler.schedule(
      HostedRefAndCache(id.toRef(), cache),
    );
    final versionInfo = versions.firstWhereOrNull(
      (i) => i.version == id.version,
    );
    final packageName = id.name;
    final version = id.version;
    if (versionInfo == null) {
      throw PackageNotFoundException(
        'Package $packageName has no version $version',
      );
    }

    late final Uint8List contentHash;
    final archiveUrl = versionInfo.archiveUrl;
    log.io('Get package from $archiveUrl.');
    log.fine('Downloading ${log.bold(id.name)} ${id.version}...');

    // Download and extract the archive to a temp directory.
    return await withTempDir((tempDirForArchive) async {
      final fileName = '$packageName-$version.tar.gz';
      final archivePath = p.join(tempDirForArchive, fileName);

      Stream<List<int>> validateSha256(
        Stream<List<int>> stream,
        Digest? expectedHash,
      ) async* {
        final output = _SingleValueSink<Digest>();
        final input = sha256.startChunkedConversion(output);
        await for (final v in stream) {
          input.add(v);
          yield v;
        }
        input.close();
        final actualHash = output.value;
        if (expectedHash != null && output.value != expectedHash) {
          log.fine(
            'Expected content-hash for ${id.name}-${id.version} $expectedHash '
            'actual: ${output.value}.',
          );
          throw PackageIntegrityException('''
Downloaded archive for ${id.name}-${id.version} had wrong content-hash.

This indicates a problem on the package repository: `${description.url}`.

See $contentHashesDocumentationUrl.
''');
        }
        contentHash = Uint8List.fromList(actualHash.bytes);
        writeHash(id, cache, contentHash);
      }

      // It is important that we do not compare against id.description.sha256,
      // as we need to check against the newly fetched version listing to ensure
      // that content changes result in updated lockfiles, not failure to
      // download.
      final expectedSha256 = versionInfo.archiveSha256;

      try {
        await withAuthenticatedClient(cache, Uri.parse(description.url), (
          client,
        ) async {
          // In addition to HTTP errors, this will retry crc32c/sha256 errors as
          // well because [PackageIntegrityException] subclasses
          // [PubHttpException].
          await retryForHttp('downloading "$archiveUrl"', () async {
            final request = http.Request('GET', archiveUrl);
            final response = await client.fetchAsStream(request);

            Stream<List<int>> stream = response.stream;
            if (_isDart2Wasm) {
              // On dart2wasm, `BrowserClient` emits JS-backed
              // `JSUint8ArrayImpl` chunks where every element read crosses FFI.
              // Copy each chunk into a WasmGC `Uint8List` once before CRC32C,
              // SHA-256, and file writing.
              stream = stream.map(Uint8List.fromList);
            }
            final expectedCrc32c = _parseCrc32c(response.headers, fileName);
            if (expectedCrc32c != null) {
              stream = _validateCrc32c(stream, expectedCrc32c, id, archiveUrl);
            }
            stream = validateSha256(
              stream,
              (expectedSha256 == null) ? null : Digest(expectedSha256),
            );

            // We download the archive to disk instead of streaming it directly
            // into the tar unpacking. This simplifies stream handling.
            // Package:tar cancels the stream when it reaches end-of-archive,
            // and cancelling a http stream makes it not reusable. There are
            // ways around this, and we might revisit this later.
            await createFileFromStream(stream, archivePath);
          });
        });
      } on Exception catch (error, stackTrace) {
        _throwFriendlyError(error, stackTrace, id.name, description.url);
      }

      try {
        await extractTarGz(readBinaryFileAsStream(archivePath), destPath);
      } on FormatException catch (e) {
        dataError('Failed to extract `$archivePath`: ${e.message}.');
      }
      return contentHash;
    });
  }

  /// Writes the contenthash for [id] in the cache.
  void writeHash(PackageId id, SystemCache cache, List<int> bytes) {
    final path = hashPath(id, cache);
    ensureDir(p.dirname(path));
    writeTextFile(path, hexEncode(bytes));
  }

  /// Installs a tar.gz file in [archivePath] as if it was downloaded from a
  /// package repository.
  ///
  /// The name, version and repository are decided from the pubspec.yaml that
  /// must be present in the archive.
  Future<PackageId> preloadPackage(
    String archivePath,
    SystemCache cache,
  ) async {
    // Extract to a temp-folder and do atomic rename to preserve the integrity
    // of the cache.
    final Uint8List contentHash;

    final tempDir = cache.createTempDir();
    final PackageId id;
    try {
      try {
        // We read the file twice, once to compute the hash, and once to extract
        // the archive.
        //
        // It would be desirable to read the file only once, but the tar
        // extraction closes the stream early making things tricky to get right.
        contentHash = Uint8List.fromList(
          (await sha256.bind(readBinaryFileAsStream(archivePath)).first).bytes,
        );
        await extractTarGz(readBinaryFileAsStream(archivePath), tempDir);
      } on FormatException catch (e) {
        dataError('Failed to extract `$archivePath`: ${e.message}.');
      }
      if (!fileExists(p.join(tempDir, 'pubspec.yaml'))) {
        fail(
          'Found no `pubspec.yaml` in $archivePath. '
          'Is it a valid pub package archive?',
        );
      }
      final Pubspec pubspec;
      try {
        pubspec = Pubspec.load(
          tempDir,
          cache.sources,
          containingDescription:
          // Dummy description. As we never use the dependencies, they don't
          // need to be resolved.
          ResolvedRootDescription.fromDir('.'),
        );
        final errors = pubspec.dependencyErrors;
        if (errors.isNotEmpty) {
          throw errors.first;
        }
      } on Exception catch (e) {
        fail('Failed to load `pubspec.yaml` from `$archivePath`: $e.');
      }
      // Reconstruct the PackageId from the extracted pubspec.yaml.
      id = PackageId(
        pubspec.name,
        pubspec.version,
        ResolvedHostedDescription(
          HostedDescription(pubspec.name, defaultUrl),
          sha256: contentHash,
        ),
      );
    } catch (e) {
      deleteEntry(tempDir);
      rethrow;
    }
    final packageDir = getDirectoryInCache(id, cache);
    if (dirExists(packageDir)) {
      log.fine(
        'Cache entry for ${id.name}-${id.version} already exists. Replacing.',
      );
      deleteEntry(packageDir);
    }
    tryRenameDir(tempDir, packageDir);
    writeHash(id, cache, contentHash);
    return id;
  }

  /// When an error occurs trying to read something about [package] from
  /// [hostedUrl], this tries to translate into a more user friendly error
  /// message.
  ///
  /// Always throws an error, either the original one or a better one.
  static Never _throwFriendlyError(
    Exception error,
    StackTrace stackTrace,
    String package,
    String hostedUrl,
  ) {
    if (error is PubHttpResponseException) {
      if (error.response.statusCode == 404) {
        throw PackageNotFoundException(
          'could not find package $package at $hostedUrl',
          innerError: error,
          innerTrace: stackTrace,
        );
      }

      fail(
        '${error.response.statusCode} ${error.response.reasonPhrase} trying '
        'to find package $package at $hostedUrl.',
        error,
        stackTrace,
      );
    } else if (error is io.SocketException) {
      fail(
        'Got socket error trying to find package $package at $hostedUrl.',
        error,
        stackTrace,
      );
    } else if (error is io.TlsException) {
      fail(
        'Got TLS error trying to find package $package at $hostedUrl.',
        error,
        stackTrace,
      );
    } else if (error is AuthenticationException) {
      String? hint;
      var message = 'authentication failed';

      assert(error.statusCode == 401 || error.statusCode == 403);
      if (error.statusCode == 401) {
        hint =
            '$hostedUrl package repository requested authentication!\n'
            'You can provide credentials using:\n'
            '    dart pub token add $hostedUrl';
      }
      if (error.statusCode == 403) {
        hint =
            'Insufficient permissions to the resource at the $hostedUrl '
            'package repository.\nYou can modify credentials using:\n'
            '    dart pub token add $hostedUrl';
        message = 'authorization failed';
      }

      if (error.serverMessage?.isNotEmpty == true && hint != null) {
        hint += '\n${error.serverMessage}';
      }

      throw PackageNotFoundException(message, hint: hint);
    } else if (error is FormatException) {
      throw PackageNotFoundException(
        'Got badly formatted response trying to find '
        'package $package at $hostedUrl',
        innerError: error,
        innerTrace: stackTrace,
        hint: 'Check that "$hostedUrl" is a valid package repository.',
      );
    } else {
      // Otherwise re-throw the original exception.
      throw error;
    }
  }

  @override
  Future<List<String>> entriesToGc(
    SystemCache cache,
    Set<String> alivePackages,
  ) async {
    final root = p.canonicalize(cache.rootDirForSource(this));
    final result = <String>{};
    final List<String> hostDirs;

    try {
      hostDirs = listDir(root);
    } on IOException {
      // Hosted cache seems uninitialized. GC nothing.
      return [];
    }
    for (final hostDir in hostDirs) {
      final List<String> packageDirs;
      try {
        packageDirs = listDir(hostDir).map(p.canonicalize).toList();
      } on IOException {
        // Failed to list `hostDir`. Perhaps a stray file? Skip.
        continue;
      }
      for (final packageDir in packageDirs) {
        if (!alivePackages.contains(packageDir)) {
          result.add(packageDir);
          // Also clear the associated hash file.
          final hashFile = p.join(
            cache.rootDir,
            'hosted-hashes',
            p.basename(hostDir),
            '${p.basename(packageDir)}.sha256',
          );
          if (fileExists(hashFile)) {
            result.add(hashFile);
          }
        }
      }
      // Clear all version listings older than two days, they'd likely need to
      // be re-fetched anyways:
      for (final cacheFile in listDir(
        p.join(hostDir, _versionListingDirectory),
      )) {
        final stat = tryStatFile(cacheFile);

        if (stat != null &&
            DateTime.now().difference(stat.modified) >
                const Duration(days: 2)) {
          result.add(cacheFile);
        }
      }
    }

    return result.toList();
  }

  /// Enables speculative prefetching of dependencies of packages queried with
  /// [doGetVersions].
  Future<T> withPrefetching<T>(
    SystemCache cache,
    Future<T> Function() callback,
  ) async {
    return await cache.hostedCache.scheduler.withPrescheduling((
      preschedule,
    ) async {
      return await runZoned(
        callback,
        zoneValues: {_prefetchingKey: preschedule},
      );
    });
  }

  /// Key for storing the current prefetch function in the current [Zone].
  static const _prefetchingKey = #_prefetch;
}

/// The [PackageRef.description] for a [HostedSource], storing the
/// [packageName] and resolved [url] of the package server.
class HostedDescription extends Description {
  final String packageName;
  final String url;

  HostedDescription._(this.packageName, this.url);

  // This can be used to construct a description with any specific url.
  factory HostedDescription.raw(String packageName, String url) =>
      HostedDescription._(packageName, url);

  factory HostedDescription(String packageName, String url) =>
      HostedDescription._(
        packageName,
        validateAndNormalizeHostedUrl(url).toString(),
      );

  @override
  int get hashCode => Object.hash(packageName, url);

  @override
  bool operator ==(Object other) {
    return other is HostedDescription &&
        other.packageName == packageName &&
        other.url == url;
  }

  @override
  String format() => 'on $url';

  @override
  Object? serializeForPubspec({
    required String? containingDir,
    required LanguageVersion languageVersion,
  }) {
    if (url == source.defaultUrl) {
      return null;
    }
    if (languageVersion >=
        LanguageVersion.firstVersionWithShorterHostedSyntax) {
      return url;
    }
    return {'url': url, 'name': packageName};
  }

  @override
  HostedSource get source => HostedSource.instance;

  @override
  bool get hasMultipleVersions => true;
}

class ResolvedHostedDescription extends ResolvedDescription {
  @override
  HostedDescription get description => super.description as HostedDescription;

  /// The content hash of the package archive (the `tar.gz` file) of the
  /// PackageId described by this.
  ///
  /// This can be obtained in several ways:
  /// * Reported from a server in the archive_sha256 field. (will be null if the
  ///   server does not report this.)
  /// * Obtained from a pubspec.lock (will be null for legacy lock-files).
  /// * Read from the
  ///   `<PUB_CACHE>/hosted-hashes/<server>/<package>-<version>.sha256` file.
  ///   (will be null if the file doesn't exist for corrupt or legacy caches).
  final Uint8List? sha256;

  ResolvedHostedDescription(
    HostedDescription super.description, {
    required this.sha256,
  });

  @override
  Object? serializeForLockfile({required String? containingDir}) {
    final hash = sha256;
    return {
      'name': description.packageName,
      'url': description.url,
      if (hash != null) 'sha256': hexEncode(hash),
    };
  }

  @override
  // We do not include the sha256 in the hashCode because of the equality
  // semantics.
  int get hashCode => description.hashCode;

  @override
  bool operator ==(Object other) {
    return other is ResolvedHostedDescription &&
        other.description == description &&
        // A [sha256] of `null` means that we don't know the hash yet.
        // Therefore we have to assume it is equal to any known value.
        (sha256 == null ||
            other.sha256 == null ||
            fixedTimeBytesEquals(sha256, other.sha256));
  }

  ResolvedHostedDescription withSha256(Uint8List? newSha256) =>
      ResolvedHostedDescription(description, sha256: newSha256);
}

/// Information about a package version retrieved from /api/packages/$package<
class HostedVersionInfo {
  final Pubspec pubspec;
  final Uri archiveUrl;
  final Version version;

  /// The sha256 digest of the archive according to the package-repository.
  final Uint8List? archiveSha256;
  final PackageStatus status;

  HostedVersionInfo(
    this.version,
    this.pubspec,
    this.archiveUrl,
    this.status,
    this.archiveSha256,
  );
}

/// Information about a security advisory affecting a package. The information
/// is retrieved from /api/packages/$package/advisories
class Advisory {
  String id;
  Set<String> affectedVersions;
  List<String> aliases;
  String summary;
  String? pubDisplayUrl;
  Advisory({
    required this.id,
    required this.affectedVersions,
    required this.summary,
    this.aliases = const [],
    this.pubDisplayUrl,
  });

  String get displayHandle => sanitizeForTerminal(pubDisplayUrl ?? id);
}

/// Given a URL, returns a "normalized" string to be used as a directory name
/// for packages downloaded from the server at that URL.
///
/// This normalization strips off the scheme (which is presumed to be HTTP or
/// HTTPS) and *sort of* URL-encodes it. I say "sort of" because it does it
/// incorrectly: it uses the character's *decimal* ASCII value instead of hex.
///
/// This could cause an ambiguity since some characters get encoded as three
/// digits and others two. It's possible for one to be a prefix of the other.
/// In practice, the set of characters that are encoded don't happen to have
/// any collisions, so the encoding is reversible.
///
/// This behavior is a bug, but is being preserved for compatibility.
String _urlToDirectory(String hostedUrl) {
  // Normalize all loopback URLs to "localhost".
  final url = hostedUrl.replaceAllMapped(
    RegExp(r'^(https?://)(127\.0\.0\.1|\[::1\]|localhost)?'),
    (match) {
      // Don't include the scheme for HTTPS URLs. This makes the directory names
      // nice for the default and most recommended scheme. We also don't include
      // it for localhost URLs, since they're always known to be HTTP.
      final localhost = match[2] == null ? '' : 'localhost';
      final scheme =
          match[1] == 'https://' || localhost.isNotEmpty ? '' : match[1];
      return '$scheme$localhost';
    },
  );
  return replace(
    url,
    RegExp(r'[<>:"\\/|?*%]'),
    (match) => '%${match[0]!.codeUnitAt(0)}',
  );
}

/// Given a directory name in the system cache, returns the URL of the server
/// whose packages it contains.
///
/// See [_urlToDirectory] for details on the mapping. Note that because the
/// directory name does not preserve the scheme, this has to guess at it. It
/// chooses "http" for loopback URLs (mainly to support the pub tests) and
/// "https" for all others.
String _directoryToUrl(String directory) {
  // Decode the pseudo-URL-encoded characters.
  const chars = '<>:"\\/|?*%';
  for (var i = 0; i < chars.length; i++) {
    final c = chars.substring(i, i + 1);
    directory = directory.replaceAll('%${c.codeUnitAt(0)}', c);
  }

  // If the URL has an explicit scheme, use that.
  if (directory.contains('://')) {
    return Uri.parse(directory).toString();
  }

  // Otherwise, default to http for localhost and https for everything else.
  final scheme =
      isLoopback(directory.replaceAll(RegExp(':.*'), '')) ? 'http' : 'https';
  return Uri.parse('$scheme://$directory').toString();
}

// TODO(sigurdm): This is quite inelegant.
class HostedRefAndCache {
  final PackageRef ref;
  final SystemCache cache;
  HostedRefAndCache(this.ref, this.cache);

  @override
  int get hashCode => ref.hashCode;
  @override
  bool operator ==(Object other) =>
      other is HostedRefAndCache && other.ref == ref;
}

/// A sink that can only have `add` called once, and that can retrieve the
/// value.
class _SingleValueSink<T> implements Sink<T> {
  late final T value;

  @override
  void add(T data) {
    value = data;
  }

  @override
  void close() {}
}

@visibleForTesting
const checksumHeaderName = 'x-goog-hash';

/// Adds a checksum validation "tap" to the response stream and returns a
/// wrapped `Stream` object, which should be used to consume the incoming data.
///
/// As chunks are received, a CRC32C checksum is updated.
/// Once the download is completed, the final checksum is compared with
/// the one present in the checksum response header.
///
/// Throws [PackageIntegrityException] if there is a checksum mismatch.
Stream<List<int>> _validateCrc32c(
  Stream<List<int>> stream,
  int expectedChecksum,
  PackageId id,
  Uri archiveUrl,
) async* {
  final crc32c = Crc32c();

  await for (final chunk in stream) {
    crc32c.update(chunk);
    yield chunk;
  }

  final actualChecksum = crc32c.finalize();

  log.fine(
    'Computed checksum $actualChecksum for ${id.name} ${id.version} with '
    'expected CRC32C of $expectedChecksum.',
  );

  if (actualChecksum != expectedChecksum) {
    throw PackageIntegrityException(
      'Package archive for ${id.name} ${id.version} downloaded from '
      '"$archiveUrl" has "x-goog-hash: crc32c=$expectedChecksum", which '
      'doesn\'t match the checksum of the archive downloaded.',
    );
  }
}

/// Parses response [headers] and returns the archive's CRC32C checksum.
///
/// In most cases, GCS provides both MD5 and CRC32C checksums in its response
/// headers. It uses the header name "x-goog-hash" for these values. It has
/// been documented and observed that GCS will send multiple response headers
/// with the same "x-goog-hash" token as the key.
/// https://cloud.google.com/storage/docs/xml-api/reference-headers#xgooghash
///
/// Additionally, when the Dart http client encounters multiple response
/// headers with the same key, it concatenates their values with a comma
/// before inserting a single item with that key and concatenated value into
/// its response "headers" Map.
/// See https://github.com/dart-lang/http/issues/24
/// https://github.com/dart-lang/http/blob/06649afbb5847dbb0293816ba8348766b116e419/pkgs/http/lib/src/base_response.dart#L29
///
/// Throws [PackageIntegrityException] if the CRC32C checksum cannot be parsed.
int? _parseCrc32c(Map<String, String> headers, String fileName) {
  final checksumHeader = headers[checksumHeaderName];
  if (checksumHeader == null) return null;

  final parts = checksumHeader.split(',');
  for (final part in parts) {
    if (part.startsWith('crc32c=')) {
      final undecoded = part.substring('crc32c='.length);

      try {
        final bytes = base64Decode(undecoded);

        // CRC32C must be 32 bits, or 4 bytes.
        if (bytes.length != 4) {
          throw FormatException('CRC32C checksum has invalid length', bytes);
        }

        return ByteData.view(bytes.buffer).getUint32(0);
      } on FormatException catch (e, s) {
        log.exception(e, s);
        throw PackageIntegrityException(
          'Package archive "$fileName" has a malformed CRC32C checksum in '
          'its response headers',
        );
      }
    }
  }

  return null;
}

/// State that is cached for the HostedSource.
final class HostedSourceCache {
  /// A pool to rate-limit concurrent network requests to pub.dev.
  final Pool pool;

  /// The scheduler used to fetch version information for packages.
  ///
  /// This allows rate-limiting requests and speculative pre-fetching of
  /// dependencies.
  final RateLimitedScheduler<HostedRefAndCache, List<HostedVersionInfo>>
  scheduler;

  /// An in-memory cache to store the cached version listing loaded from
  /// [HostedSource._versionListingCachePath].
  ///
  /// Invariant: Entries in this cache are the parsed version of the exact same
  /// information cached on disk. I.e. if the entry is present in this cache,
  /// there will not be a newer version on disk.
  final Map<PackageRef, (DateTime, List<HostedVersionInfo>)> _responseCache =
      {};

  /// An in-memory cache to store the futures of fetched security advisories.
  final Map<PackageId, Future<List<Advisory>?>> _advisoriesCache = {};

  HostedSourceCache(HostedSource hosted) : this._(hosted, Pool(10));

  HostedSourceCache._(HostedSource hosted, this.pool)
    : scheduler = RateLimitedScheduler(
        (HostedRefAndCache refAndCache) => hosted.fetchVersions(refAndCache),
        pool: pool,
      );
}