blob: 41797ffad34a7126c08535ed2103ad1a561f59dd [file] [log] [blame]
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io' as io;
import 'package:collection/collection.dart' show maxBy;
import 'package:http/http.dart' as http;
import 'package:path/path.dart' as p;
import 'package:pedantic/pedantic.dart';
import 'package:pub_semver/pub_semver.dart';
import 'package:stack_trace/stack_trace.dart';
import '../exceptions.dart';
import '../http.dart';
import '../io.dart';
import '../log.dart' as log;
import '../package.dart';
import '../package_name.dart';
import '../pubspec.dart';
import '../rate_limited_scheduler.dart';
import '../source.dart';
import '../system_cache.dart';
import '../utils.dart';
import 'cached.dart';
/// A package source that gets packages from a package hosting site that uses
/// the same API as pub.dartlang.org.
class HostedSource extends Source {
@override
final name = 'hosted';
@override
final hasMultipleVersions = true;
@override
BoundHostedSource bind(SystemCache systemCache, {bool isOffline = false}) =>
isOffline
? _OfflineHostedSource(this, systemCache)
: BoundHostedSource(this, systemCache);
/// Gets the default URL for the package server for hosted dependencies.
String get defaultUrl {
// Changing this to pub.dev raises the following concerns:
//
// 1. It would blow through users caches.
// 2. It would cause conflicts for users checking pubspec.lock into git, if using
// different versions of the dart-sdk / pub client.
// 3. It might cause other problems (investigation needed) for pubspec.lock across
// different versions of the dart-sdk / pub client.
// 4. It would expand the API surface we're committed to supporting long-term.
//
// Clearly, a bit of investigation is necessary before we update this to
// pub.dev, it might be attractive to do next time we change the server API.
return _defaultUrl ??= _pubHostedUrlConfig() ?? 'https://pub.dartlang.org';
}
String _defaultUrl;
String _pubHostedUrlConfig() {
var url = io.Platform.environment['PUB_HOSTED_URL'];
if (url == null) return null;
var uri = Uri.parse(url);
if (!uri.isScheme('http') && !uri.isScheme('https')) {
throw ConfigException(
'`PUB_HOSTED_URL` must have either the scheme "https://" or "http://". '
'"$url" is invalid.');
}
return url;
}
/// Returns a reference to a hosted package named [name].
///
/// If [url] is passed, it's the URL of the pub server from which the package
/// should be downloaded. It can be a [Uri] or a [String].
PackageRef refFor(String name, {url}) =>
PackageRef(name, this, _descriptionFor(name, url));
/// Returns an ID for a hosted package named [name] at [version].
///
/// If [url] is passed, it's the URL of the pub server from which the package
/// should be downloaded. It can be a [Uri] or a [String].
PackageId idFor(String name, Version version, {url}) =>
PackageId(name, this, version, _descriptionFor(name, url));
/// Returns the description for a hosted package named [name] with the
/// given package server [url].
dynamic _descriptionFor(String name, [url]) {
if (url == null) return name;
if (url is! String && url is! Uri) {
throw ArgumentError.value(url, 'url', 'must be a Uri or a String.');
}
return {'name': name, 'url': url.toString()};
}
@override
String formatDescription(description) =>
'on ${_parseDescription(description).last}';
@override
bool descriptionsEqual(description1, description2) =>
_parseDescription(description1) == _parseDescription(description2);
@override
int hashDescription(description) => _parseDescription(description).hashCode;
/// Ensures that [description] is a valid hosted package description.
///
/// There are two valid formats. A plain string refers to a package with the
/// given name from the default host, while a map with keys "name" and "url"
/// refers to a package with the given name from the host at the given URL.
@override
PackageRef parseRef(String name, description, {String containingPath}) {
_parseDescription(description);
return PackageRef(name, this, description);
}
@override
PackageId parseId(String name, Version version, description,
{String containingPath}) {
_parseDescription(description);
return PackageId(name, this, version, description);
}
/// Parses the description for a package.
///
/// If the package parses correctly, this returns a (name, url) pair. If not,
/// this throws a descriptive FormatException.
Pair<String, String> _parseDescription(description) {
if (description is String) {
return Pair<String, String>(description, defaultUrl);
}
if (description is! Map) {
throw FormatException('The description must be a package name or map.');
}
if (!description.containsKey('name')) {
throw FormatException("The description map must contain a 'name' key.");
}
var name = description['name'];
if (name is! String) {
throw FormatException("The 'name' key must have a string value.");
}
return Pair<String, String>(name, description['url'] ?? defaultUrl);
}
}
/// Information about a package version retrieved from /api/packages/$package
class _VersionInfo {
final Pubspec pubspec;
final Uri archiveUrl;
final PackageStatus status;
_VersionInfo(this.pubspec, this.archiveUrl, this.status);
}
/// The [BoundSource] for [HostedSource].
class BoundHostedSource extends CachedSource {
@override
final HostedSource source;
@override
final SystemCache systemCache;
RateLimitedScheduler<PackageRef, Map<PackageId, _VersionInfo>> _scheduler;
BoundHostedSource(this.source, this.systemCache) {
_scheduler = RateLimitedScheduler(
_fetchVersions,
maxConcurrentOperations: 10,
);
}
Map<PackageId, _VersionInfo> _versionInfoFromPackageListing(
Map body, PackageRef ref, Uri location) {
final versions = body['versions'] as List;
return Map.fromEntries(versions.map((map) {
var pubspec = Pubspec.fromMap(map['pubspec'], systemCache.sources,
expectedName: ref.name, location: location);
var id = source.idFor(ref.name, pubspec.version,
url: _serverFor(ref.description));
final archiveUrlValue = map['archive_url'];
final archiveUrl =
archiveUrlValue is String ? Uri.tryParse(archiveUrlValue) : null;
final status = PackageStatus(
isDiscontinued: body['isDiscontinued'] as bool ?? false,
discontinuedReplacedBy: body['replacedBy'] as String);
return MapEntry(id, _VersionInfo(pubspec, archiveUrl, status));
}));
}
Future<Map<PackageId, _VersionInfo>> _fetchVersions(PackageRef ref) async {
var url = _makeUrl(
ref.description, (server, package) => '$server/api/packages/$package');
log.io('Get versions from $url.');
String bodyText;
Map body;
try {
// TODO(sigurdm): Implement cancellation of requests. This probably
// requires resolution of: https://github.com/dart-lang/sdk/issues/22265.
bodyText = await httpClient.read(url, headers: pubApiHeaders);
body = jsonDecode(bodyText);
} catch (error, stackTrace) {
var parsed = source._parseDescription(ref.description);
_throwFriendlyError(error, stackTrace, parsed.first, parsed.last);
}
final result = _versionInfoFromPackageListing(body, ref, url);
// Cache the response on disk.
// Don't cache overly big responses.
if (body.length < 100 * 1024) {
await _cacheVersionListingResponse(body, ref);
}
// Prefetch the dependencies of the latest version, we are likely to need
// them later.
final preschedule =
Zone.current[_prefetchingKey] as void Function(PackageRef);
if (preschedule != null) {
final latestVersion =
maxBy(result.keys.map((id) => id.version), (e) => e);
final latestVersionId =
PackageId(ref.name, source, latestVersion, ref.description);
final dependencies =
result[latestVersionId]?.pubspec?.dependencies?.values ?? [];
unawaited(withDependencyType(DependencyType.none, () async {
for (final packageRange in dependencies) {
if (packageRange.source is HostedSource) {
preschedule(packageRange.toRef());
}
}
}));
}
return result;
}
/// If a cached version listing response for [ref] exists on disk and is less
/// than [maxAge] old it is parsed and returned.
///
/// Otherwise deletes a cached response if it exists and returns `null`.
Future<Map<PackageId, _VersionInfo>> _cachedVersionListingResponse(
PackageRef ref, Duration maxAge) async {
final cachePath = _versionListingCachePath(ref);
final stat = await io.File(cachePath).stat();
final now = DateTime.now();
if (stat.type == io.FileSystemEntityType.file) {
if (now.difference(stat.modified) < maxAge) {
try {
final cachedDoc = jsonDecode(await readTextFileAsync(cachePath));
final timestamp = cachedDoc['_fetchedAt'];
if (timestamp is String) {
final cacheAge =
DateTime.now().difference(DateTime.parse(timestamp));
if (cacheAge > maxAge) {
// Too old according to internal timestamp - delete.
tryDeleteEntry(cachePath);
} else {
return _versionInfoFromPackageListing(
cachedDoc, ref, Uri.file(cachePath));
}
}
} on io.IOException {
// Could not read the file. Delete if it exists.
tryDeleteEntry(cachePath);
} on FormatException {
// Decoding error - bad file or bad timestamp. Delete the file.
tryDeleteEntry(cachePath);
}
} else {
// File too old
tryDeleteEntry(cachePath);
}
}
return null;
}
/// Saves the (decoded) response from package-listing of [ref].
Future<void> _cacheVersionListingResponse(Map body, PackageRef ref) async {
final path = _versionListingCachePath(ref);
try {
ensureDir(p.dirname(path));
await writeTextFileAsync(
path,
jsonEncode(
<String, dynamic>{
...body,
'_fetchedAt': DateTime.now().toIso8601String(),
},
),
);
} on io.IOException catch (e) {
// Not being able to write this cache is not fatal. Just move on...
log.fine('Failed writing cache file. $e');
}
}
@override
Future<PackageStatus> status(PackageId id, Duration maxAge) async {
final ref = id.toRef();
// Did we already get info for this package?
var versionListing = _scheduler.peek(ref);
// Do we have a cached version response on disk?
versionListing ??= await _cachedVersionListingResponse(ref, maxAge);
// Otherwise retrieve the info from the host.
versionListing ??= await _scheduler
.schedule(ref)
// Failures retrieving the listing here should just be ignored.
.catchError(
(_) => <PackageId, _VersionInfo>{},
test: (error) => error is Exception,
);
final listing = versionListing[id];
// If we don't have the specific version we return the empty response.
//
// This should not happen. But in production we want to avoid a crash, since
// it is more or less harmless.
//
// TODO(sigurdm): Consider representing the non-existence of the
// package-version in the return value.
assert(listing != null);
return versionListing[id]?.status ?? PackageStatus();
}
// The path where the response from the package-listing api is cached.
String _versionListingCachePath(PackageRef ref) {
final parsed = source._parseDescription(ref.description);
final dir = _urlToDirectory(parsed.last);
// Use a dot-dir because older versions of pub won't choke on that
// name when iterating the cache (it is not listed by [listDir]).
return p.join(systemCacheRoot, dir, _versionListingDirectory,
'${ref.name}-versions.json');
}
static const _versionListingDirectory = '.cache';
/// Downloads a list of all versions of a package that are available from the
/// site.
@override
Future<List<PackageId>> doGetVersions(PackageRef ref, Duration maxAge) async {
Map<PackageId, _VersionInfo> versionListing;
if (maxAge != null) {
// Do we have a cached version response on disk?
versionListing ??= await _cachedVersionListingResponse(ref, maxAge);
}
versionListing ??= await _scheduler.schedule(ref);
return versionListing.keys.toList();
}
/// Parses [description] into its server and package name components, then
/// converts that to a Uri given [pattern].
///
/// Ensures the package name is properly URL encoded.
Uri _makeUrl(
description, String Function(String server, String package) pattern) {
var parsed = source._parseDescription(description);
var server = parsed.last;
var package = Uri.encodeComponent(parsed.first);
return Uri.parse(pattern(server, package));
}
/// Retrieves the pubspec for a specific version of a package that is
/// available from the site.
@override
Future<Pubspec> describeUncached(PackageId id) async {
final versions = await _scheduler.schedule(id.toRef());
final url = _makeUrl(
id.description, (server, package) => '$server/api/packages/$package');
return versions[id]?.pubspec ??
(throw PackageNotFoundException('Could not find package $id at $url'));
}
/// Downloads the package identified by [id] to the system cache.
@override
Future<Package> downloadToSystemCache(PackageId id) async {
if (!isInSystemCache(id)) {
var packageDir = getDirectory(id);
ensureDir(p.dirname(packageDir));
await _download(id, packageDir);
}
return Package.load(id.name, getDirectory(id), systemCache.sources);
}
/// The system cache directory for the hosted source contains subdirectories
/// for each separate repository URL that's used on the system.
///
/// Each of these subdirectories then contains a subdirectory for each
/// package downloaded from that site.
@override
String getDirectory(PackageId id) {
var parsed = source._parseDescription(id.description);
var dir = _urlToDirectory(parsed.last);
return p.join(systemCacheRoot, dir, '${parsed.first}-${id.version}');
}
/// Re-downloads all packages that have been previously downloaded into the
/// system cache from any server.
@override
Future<Iterable<RepairResult>> repairCachedPackages() async {
if (!dirExists(systemCacheRoot)) return [];
return (await Future.wait(listDir(systemCacheRoot).map(
(serverDir) async {
var url = _directoryToUrl(p.basename(serverDir));
final results = <RepairResult>[];
var packages = <Package>[];
for (var entry in listDir(serverDir)) {
try {
packages.add(Package.load(null, entry, systemCache.sources));
} catch (error, stackTrace) {
log.error('Failed to load package', error, stackTrace);
results.add(
RepairResult(
_idForBasename(
p.basename(entry),
url: _directoryToUrl(serverDir),
),
success: false,
),
);
tryDeleteEntry(entry);
}
}
// Delete the cached package listings.
tryDeleteEntry(p.join(serverDir, _versionListingDirectory));
packages.sort(Package.orderByNameAndVersion);
return results
..addAll(await Future.wait(
packages.map(
(package) async {
var id = source.idFor(package.name, package.version, url: url);
try {
await _download(id, package.dir);
return RepairResult(id, success: true);
} catch (error, stackTrace) {
var message = 'Failed to repair ${log.bold(package.name)} '
'${package.version}';
if (url != source.defaultUrl) message += ' from $url';
log.error('$message. Error:\n$error');
log.fine(stackTrace);
tryDeleteEntry(package.dir);
return RepairResult(id, success: false);
}
},
),
));
},
)))
.expand((x) => x);
}
/// Returns the best-guess package ID for [basename], which should be a
/// subdirectory in a hosted cache.
PackageId _idForBasename(String basename, {String url}) {
var components = split1(basename, '-');
var version = Version.none;
if (components.length > 1) {
try {
version = Version.parse(components.last);
} catch (_) {
// Default to Version.none.
}
}
final name = components.first;
return source.idFor(name, version, url: url);
}
bool _looksLikePackageDir(String path) =>
dirExists(path) &&
_idForBasename(p.basename(path)).version != Version.none;
/// Gets all of the packages that have been downloaded into the system cache
/// from the default server.
@override
List<Package> getCachedPackages() {
var cacheDir = p.join(systemCacheRoot, _urlToDirectory(source.defaultUrl));
if (!dirExists(cacheDir)) return [];
return listDir(cacheDir)
.where(_looksLikePackageDir)
.map((entry) {
try {
return Package.load(null, entry, systemCache.sources);
} catch (error, stackTrace) {
log.fine('Failed to load package from $entry:\n'
'$error\n'
'${Chain.forTrace(stackTrace)}');
return null;
}
})
.where((e) => e != null)
.toList();
}
/// Downloads package [package] at [version] from the archive_url and unpacks
/// it into [destPath].
///
/// If there is no archive_url, try to fetch it from
/// `$server/packages/$package/versions/$version.tar.gz` where server comes
/// from `id.description`.
Future _download(PackageId id, String destPath) async {
final versions = await _scheduler.schedule(id.toRef());
final versionInfo = versions[id];
final packageName = id.name;
final version = id.version;
if (versionInfo == null) {
throw PackageNotFoundException(
'Package $packageName has no version $version');
}
var url = versionInfo.archiveUrl;
if (url == null) {
// To support old servers that has no archive_url we fall back to the
// hard-coded path.
final parsedDescription = source._parseDescription(id.description);
final server = parsedDescription.last;
url = Uri.parse('$server/packages/$packageName/versions/$version.tar.gz');
}
log.io('Get package from $url.');
log.message('Downloading ${log.bold(id.name)} ${id.version}...');
// Download and extract the archive to a temp directory.
var tempDir = systemCache.createTempDir();
var response = await httpClient.send(http.Request('GET', url));
await extractTarGz(response.stream, tempDir);
// Remove the existing directory if it exists. This will happen if
// we're forcing a download to repair the cache.
if (dirExists(destPath)) deleteEntry(destPath);
// Now that the get has succeeded, move it to the real location in the
// cache. This ensures that we don't leave half-busted ghost
// directories in the user's pub cache if a get fails.
renameDir(tempDir, destPath);
}
/// When an error occurs trying to read something about [package] from [url],
/// this tries to translate into a more user friendly error message.
///
/// Always throws an error, either the original one or a better one.
void _throwFriendlyError(
error, StackTrace stackTrace, String package, String url) {
if (error is PubHttpException) {
if (error.response.statusCode == 404) {
throw PackageNotFoundException(
'could not find package $package at $url',
innerError: error,
innerTrace: stackTrace);
}
fail(
'${error.response.statusCode} ${error.response.reasonPhrase} trying '
'to find package $package at $url.',
error,
stackTrace);
} else if (error is io.SocketException) {
fail('Got socket error trying to find package $package at $url.', error,
stackTrace);
} else if (error is io.TlsException) {
fail('Got TLS error trying to find package $package at $url.', error,
stackTrace);
} else {
// Otherwise re-throw the original exception.
throw error;
}
}
/// Given a URL, returns a "normalized" string to be used as a directory name
/// for packages downloaded from the server at that URL.
///
/// This normalization strips off the scheme (which is presumed to be HTTP or
/// HTTPS) and *sort of* URL-encodes it. I say "sort of" because it does it
/// incorrectly: it uses the character's *decimal* ASCII value instead of hex.
///
/// This could cause an ambiguity since some characters get encoded as three
/// digits and others two. It's possible for one to be a prefix of the other.
/// In practice, the set of characters that are encoded don't happen to have
/// any collisions, so the encoding is reversible.
///
/// This behavior is a bug, but is being preserved for compatibility.
String _urlToDirectory(String url) {
// Normalize all loopback URLs to "localhost".
url = url.replaceAllMapped(
RegExp(r'^(https?://)(127\.0\.0\.1|\[::1\]|localhost)?'), (match) {
// Don't include the scheme for HTTPS URLs. This makes the directory names
// nice for the default and most recommended scheme. We also don't include
// it for localhost URLs, since they're always known to be HTTP.
var localhost = match[2] == null ? '' : 'localhost';
var scheme =
match[1] == 'https://' || localhost.isNotEmpty ? '' : match[1];
return '$scheme$localhost';
});
return replace(
url, RegExp(r'[<>:"\\/|?*%]'), (match) => '%${match[0].codeUnitAt(0)}');
}
/// Given a directory name in the system cache, returns the URL of the server
/// whose packages it contains.
///
/// See [_urlToDirectory] for details on the mapping. Note that because the
/// directory name does not preserve the scheme, this has to guess at it. It
/// chooses "http" for loopback URLs (mainly to support the pub tests) and
/// "https" for all others.
String _directoryToUrl(String url) {
// Decode the pseudo-URL-encoded characters.
var chars = '<>:"\\/|?*%';
for (var i = 0; i < chars.length; i++) {
var c = chars.substring(i, i + 1);
url = url.replaceAll('%${c.codeUnitAt(0)}', c);
}
// If the URL has an explicit scheme, use that.
if (url.contains('://')) return url;
// Otherwise, default to http for localhost and https for everything else.
var scheme =
isLoopback(url.replaceAll(RegExp(':.*'), '')) ? 'http' : 'https';
return '$scheme://$url';
}
/// Returns the server URL for [description].
Uri _serverFor(description) =>
Uri.parse(source._parseDescription(description).last);
/// Enables speculative prefetching of dependencies of packages queried with
/// [getVersions].
Future<T> withPrefetching<T>(Future<T> Function() callback) async {
return await _scheduler.withPrescheduling((preschedule) async {
return await runZoned(callback,
zoneValues: {_prefetchingKey: preschedule});
});
}
/// Key for storing the current prefetch function in the current [Zone].
static const _prefetchingKey = #_prefetch;
}
/// This is the modified hosted source used when pub get or upgrade are run
/// with "--offline".
///
/// This uses the system cache to get the list of available packages and does
/// no network access.
class _OfflineHostedSource extends BoundHostedSource {
_OfflineHostedSource(HostedSource source, SystemCache systemCache)
: super(source, systemCache);
/// Gets the list of all versions of [ref] that are in the system cache.
@override
Future<List<PackageId>> doGetVersions(PackageRef ref, Duration maxAge) async {
var parsed = source._parseDescription(ref.description);
var server = parsed.last;
log.io('Finding versions of ${ref.name} in '
'$systemCacheRoot/${_urlToDirectory(server)}');
var dir = p.join(systemCacheRoot, _urlToDirectory(server));
List<PackageId> versions;
if (dirExists(dir)) {
versions = listDir(dir)
.where(_looksLikePackageDir)
.map((entry) => _idForBasename(p.basename(entry), url: server))
.where((id) => id.name == ref.name && id.version != Version.none)
.toList();
} else {
versions = [];
}
// If there are no versions in the cache, report a clearer error.
if (versions.isEmpty) {
throw PackageNotFoundException(
'could not find package ${ref.name} in cache');
}
return versions;
}
@override
Future _download(PackageId id, String destPath) {
// Since HostedSource is cached, this will only be called for uncached
// packages.
throw UnsupportedError('Cannot download packages when offline.');
}
@override
Future<Pubspec> describeUncached(PackageId id) {
throw PackageNotFoundException(
'${id.name} ${id.version} is not available in your system cache');
}
@override
Future<PackageStatus> status(PackageId id, Duration maxAge) async {
// Do we have a cached version response on disk?
final versionListing =
await _cachedVersionListingResponse(id.toRef(), maxAge);
if (versionListing == null) {
return PackageStatus();
}
final listing = versionListing[id];
// If we don't have the specific version we return the empty response.
//
// This should not happen. But in production we want to avoid a crash, since
// it is more or less harmless.
//
// TODO(sigurdm): Consider representing the non-existence of the
// package-version in the return value.
assert(listing != null);
return versionListing[id]?.status ?? PackageStatus();
}
}