| // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'dart:async'; |
| import "dart:convert"; |
| import 'dart:io' as io; |
| |
| import 'package:http/http.dart' as http; |
| import 'package:path/path.dart' as p; |
| import 'package:pub_semver/pub_semver.dart'; |
| import 'package:stack_trace/stack_trace.dart'; |
| |
| import '../exceptions.dart'; |
| import '../http.dart'; |
| import '../io.dart'; |
| import '../log.dart' as log; |
| import '../package.dart'; |
| import '../package_name.dart'; |
| import '../pubspec.dart'; |
| import '../source.dart'; |
| import '../system_cache.dart'; |
| import '../utils.dart'; |
| import 'cached.dart'; |
| |
| /// A package source that gets packages from a package hosting site that uses |
| /// the same API as pub.dartlang.org. |
| class HostedSource extends Source { |
| final name = "hosted"; |
| final hasMultipleVersions = true; |
| |
| BoundHostedSource bind(SystemCache systemCache, {bool isOffline: false}) => |
| isOffline |
| ? new _OfflineHostedSource(this, systemCache) |
| : new BoundHostedSource(this, systemCache); |
| |
| /// Gets the default URL for the package server for hosted dependencies. |
| String get defaultUrl { |
| var url = io.Platform.environment["PUB_HOSTED_URL"]; |
| if (url != null) return url; |
| |
| return "https://pub.dartlang.org"; |
| } |
| |
| /// Returns a reference to a hosted package named [name]. |
| /// |
| /// If [url] is passed, it's the URL of the pub server from which the package |
| /// should be downloaded. It can be a [Uri] or a [String]. |
| PackageRef refFor(String name, {url}) => |
| new PackageRef(name, this, _descriptionFor(name, url)); |
| |
| /// Returns an ID for a hosted package named [name] at [version]. |
| /// |
| /// If [url] is passed, it's the URL of the pub server from which the package |
| /// should be downloaded. It can be a [Uri] or a [String]. |
| PackageId idFor(String name, Version version, {url}) => |
| new PackageId(name, this, version, _descriptionFor(name, url)); |
| |
| /// Returns the description for a hosted package named [name] with the |
| /// given package server [url]. |
| _descriptionFor(String name, [url]) { |
| if (url == null) return name; |
| |
| if (url is! String && url is! Uri) { |
| throw new ArgumentError.value(url, 'url', 'must be a Uri or a String.'); |
| } |
| |
| return {'name': name, 'url': url.toString()}; |
| } |
| |
| String formatDescription(description) => |
| "on ${_parseDescription(description).last}"; |
| |
| bool descriptionsEqual(description1, description2) => |
| _parseDescription(description1) == _parseDescription(description2); |
| |
| int hashDescription(description) => _parseDescription.hashCode; |
| |
| /// Ensures that [description] is a valid hosted package description. |
| /// |
| /// There are two valid formats. A plain string refers to a package with the |
| /// given name from the default host, while a map with keys "name" and "url" |
| /// refers to a package with the given name from the host at the given URL. |
| PackageRef parseRef(String name, description, {String containingPath}) { |
| _parseDescription(description); |
| return new PackageRef(name, this, description); |
| } |
| |
| PackageId parseId(String name, Version version, description, |
| {String containingPath}) { |
| _parseDescription(description); |
| return new PackageId(name, this, version, description); |
| } |
| |
| /// Parses the description for a package. |
| /// |
| /// If the package parses correctly, this returns a (name, url) pair. If not, |
| /// this throws a descriptive FormatException. |
| Pair<String, String> _parseDescription(description) { |
| if (description is String) { |
| return new Pair<String, String>(description, defaultUrl); |
| } |
| |
| if (description is! Map) { |
| throw new FormatException( |
| "The description must be a package name or map."); |
| } |
| |
| if (!description.containsKey("name")) { |
| throw new FormatException( |
| "The description map must contain a 'name' key."); |
| } |
| |
| var name = description["name"]; |
| if (name is! String) { |
| throw new FormatException("The 'name' key must have a string value."); |
| } |
| |
| return new Pair<String, String>(name, description["url"] ?? defaultUrl); |
| } |
| } |
| |
| /// The [BoundSource] for [HostedSource]. |
| class BoundHostedSource extends CachedSource { |
| final HostedSource source; |
| |
| final SystemCache systemCache; |
| |
| BoundHostedSource(this.source, this.systemCache); |
| |
| /// Downloads a list of all versions of a package that are available from the |
| /// site. |
| Future<List<PackageId>> doGetVersions(PackageRef ref) async { |
| var url = _makeUrl( |
| ref.description, (server, package) => "$server/api/packages/$package"); |
| |
| log.io("Get versions from $url."); |
| |
| var body; |
| try { |
| body = await httpClient.read(url, headers: PUB_API_HEADERS); |
| } catch (error, stackTrace) { |
| var parsed = source._parseDescription(ref.description); |
| _throwFriendlyError(error, stackTrace, parsed.first, parsed.last); |
| } |
| |
| var doc = jsonDecode(body); |
| return (doc['versions'] as List).map((map) { |
| var pubspec = new Pubspec.fromMap(map['pubspec'], systemCache.sources, |
| expectedName: ref.name, location: url); |
| var id = source.idFor(ref.name, pubspec.version, |
| url: _serverFor(ref.description)); |
| memoizePubspec(id, pubspec); |
| |
| return id; |
| }).toList(); |
| } |
| |
| /// Parses [description] into its server and package name components, then |
| /// converts that to a Uri given [pattern]. |
| /// |
| /// Ensures the package name is properly URL encoded. |
| Uri _makeUrl(description, String pattern(String server, String package)) { |
| var parsed = source._parseDescription(description); |
| var server = parsed.last; |
| var package = Uri.encodeComponent(parsed.first); |
| return Uri.parse(pattern(server, package)); |
| } |
| |
| /// Downloads and parses the pubspec for a specific version of a package that |
| /// is available from the site. |
| Future<Pubspec> describeUncached(PackageId id) async { |
| // Request it from the server. |
| var url = _makeVersionUrl( |
| id, |
| (server, package, version) => |
| "$server/api/packages/$package/versions/$version"); |
| |
| log.io("Describe package at $url."); |
| var version; |
| try { |
| version = |
| jsonDecode(await httpClient.read(url, headers: PUB_API_HEADERS)); |
| } catch (error, stackTrace) { |
| var parsed = source._parseDescription(id.description); |
| _throwFriendlyError(error, stackTrace, id.name, parsed.last); |
| } |
| |
| return new Pubspec.fromMap(version['pubspec'], systemCache.sources, |
| expectedName: id.name, location: url); |
| } |
| |
| /// Downloads the package identified by [id] to the system cache. |
| Future<Package> downloadToSystemCache(PackageId id) async { |
| if (!isInSystemCache(id)) { |
| var packageDir = getDirectory(id); |
| ensureDir(p.dirname(packageDir)); |
| var parsed = source._parseDescription(id.description); |
| await _download(parsed.last, parsed.first, id.version, packageDir); |
| } |
| |
| return new Package.load(id.name, getDirectory(id), systemCache.sources); |
| } |
| |
| /// The system cache directory for the hosted source contains subdirectories |
| /// for each separate repository URL that's used on the system. |
| /// |
| /// Each of these subdirectories then contains a subdirectory for each |
| /// package downloaded from that site. |
| String getDirectory(PackageId id) { |
| var parsed = source._parseDescription(id.description); |
| var dir = _urlToDirectory(parsed.last); |
| return p.join(systemCacheRoot, dir, "${parsed.first}-${id.version}"); |
| } |
| |
| /// Re-downloads all packages that have been previously downloaded into the |
| /// system cache from any server. |
| Future<Pair<List<PackageId>, List<PackageId>>> repairCachedPackages() async { |
| if (!dirExists(systemCacheRoot)) return new Pair([], []); |
| |
| var successes = <PackageId>[]; |
| var failures = <PackageId>[]; |
| |
| for (var serverDir in listDir(systemCacheRoot)) { |
| var url = _directoryToUrl(p.basename(serverDir)); |
| |
| var packages = <Package>[]; |
| for (var entry in listDir(serverDir)) { |
| try { |
| packages.add(new Package.load(null, entry, systemCache.sources)); |
| } catch (error, stackTrace) { |
| log.error("Failed to load package", error, stackTrace); |
| failures.add(_idForBasename(p.basename(entry))); |
| tryDeleteEntry(entry); |
| } |
| } |
| |
| packages.sort(Package.orderByNameAndVersion); |
| |
| for (var package in packages) { |
| var id = source.idFor(package.name, package.version, url: url); |
| |
| try { |
| await _download(url, package.name, package.version, package.dir); |
| successes.add(id); |
| } catch (error, stackTrace) { |
| failures.add(id); |
| var message = "Failed to repair ${log.bold(package.name)} " |
| "${package.version}"; |
| if (url != source.defaultUrl) message += " from $url"; |
| log.error("$message. Error:\n$error"); |
| log.fine(stackTrace); |
| |
| tryDeleteEntry(package.dir); |
| } |
| } |
| } |
| |
| return new Pair(successes, failures); |
| } |
| |
| /// Returns the best-guess package ID for [basename], which should be a |
| /// subdirectory in a hosted cache. |
| PackageId _idForBasename(String basename) { |
| var components = split1(basename, '-'); |
| var version = Version.none; |
| if (components.length > 1) { |
| try { |
| version = new Version.parse(components.last); |
| } catch (_) { |
| // Default to Version.none. |
| } |
| } |
| return new PackageId(components.first, source, version, components.first); |
| } |
| |
| /// Gets all of the packages that have been downloaded into the system cache |
| /// from the default server. |
| List<Package> getCachedPackages() { |
| var cacheDir = p.join(systemCacheRoot, _urlToDirectory(source.defaultUrl)); |
| if (!dirExists(cacheDir)) return []; |
| |
| return listDir(cacheDir).map((entry) { |
| try { |
| return new Package.load(null, entry, systemCache.sources); |
| } catch (error, stackTrace) { |
| log.fine("Failed to load package from $entry:\n" |
| "$error\n" |
| "${new Chain.forTrace(stackTrace)}"); |
| } |
| }).toList(); |
| } |
| |
| /// Downloads package [package] at [version] from [server], and unpacks it |
| /// into [destPath]. |
| Future _download( |
| String server, String package, Version version, String destPath) async { |
| var url = Uri.parse("$server/packages/$package/versions/$version.tar.gz"); |
| log.io("Get package from $url."); |
| log.message('Downloading ${log.bold(package)} ${version}...'); |
| |
| // Download and extract the archive to a temp directory. |
| var tempDir = systemCache.createTempDir(); |
| var response = await httpClient.send(new http.Request("GET", url)); |
| await extractTarGz(response.stream, tempDir); |
| |
| // Remove the existing directory if it exists. This will happen if |
| // we're forcing a download to repair the cache. |
| if (dirExists(destPath)) deleteEntry(destPath); |
| |
| // Now that the get has succeeded, move it to the real location in the |
| // cache. This ensures that we don't leave half-busted ghost |
| // directories in the user's pub cache if a get fails. |
| renameDir(tempDir, destPath); |
| } |
| |
| /// When an error occurs trying to read something about [package] from [url], |
| /// this tries to translate into a more user friendly error message. |
| /// |
| /// Always throws an error, either the original one or a better one. |
| void _throwFriendlyError( |
| error, StackTrace stackTrace, String package, String url) { |
| if (error is PubHttpException) { |
| if (error.response.statusCode == 404) { |
| throw new PackageNotFoundException( |
| "could not find package $package at $url", |
| innerError: error, |
| innerTrace: stackTrace); |
| } |
| |
| fail( |
| "${error.response.statusCode} ${error.response.reasonPhrase} trying " |
| "to find package $package at $url.", |
| error, |
| stackTrace); |
| } else if (error is io.SocketException) { |
| fail("Got socket error trying to find package $package at $url.", error, |
| stackTrace); |
| } else if (error is io.TlsException) { |
| fail("Got TLS error trying to find package $package at $url.", error, |
| stackTrace); |
| } else { |
| // Otherwise re-throw the original exception. |
| throw error; |
| } |
| } |
| |
| /// Given a URL, returns a "normalized" string to be used as a directory name |
| /// for packages downloaded from the server at that URL. |
| /// |
| /// This normalization strips off the scheme (which is presumed to be HTTP or |
| /// HTTPS) and *sort of* URL-encodes it. I say "sort of" because it does it |
| /// incorrectly: it uses the character's *decimal* ASCII value instead of hex. |
| /// |
| /// This could cause an ambiguity since some characters get encoded as three |
| /// digits and others two. It's possible for one to be a prefix of the other. |
| /// In practice, the set of characters that are encoded don't happen to have |
| /// any collisions, so the encoding is reversible. |
| /// |
| /// This behavior is a bug, but is being preserved for compatibility. |
| String _urlToDirectory(String url) { |
| // Normalize all loopback URLs to "localhost". |
| url = url.replaceAllMapped( |
| new RegExp(r"^(https?://)(127\.0\.0\.1|\[::1\]|localhost)?"), (match) { |
| // Don't include the scheme for HTTPS URLs. This makes the directory names |
| // nice for the default and most recommended scheme. We also don't include |
| // it for localhost URLs, since they're always known to be HTTP. |
| var localhost = match[2] == null ? '' : 'localhost'; |
| var scheme = |
| match[1] == 'https://' || localhost.isNotEmpty ? '' : match[1]; |
| return "$scheme$localhost"; |
| }); |
| return replace(url, new RegExp(r'[<>:"\\/|?*%]'), |
| (match) => '%${match[0].codeUnitAt(0)}'); |
| } |
| |
| /// Given a directory name in the system cache, returns the URL of the server |
| /// whose packages it contains. |
| /// |
| /// See [_urlToDirectory] for details on the mapping. Note that because the |
| /// directory name does not preserve the scheme, this has to guess at it. It |
| /// chooses "http" for loopback URLs (mainly to support the pub tests) and |
| /// "https" for all others. |
| String _directoryToUrl(String url) { |
| // Decode the pseudo-URL-encoded characters. |
| var chars = '<>:"\\/|?*%'; |
| for (var i = 0; i < chars.length; i++) { |
| var c = chars.substring(i, i + 1); |
| url = url.replaceAll("%${c.codeUnitAt(0)}", c); |
| } |
| |
| // If the URL has an explicit scheme, use that. |
| if (url.contains("://")) return url; |
| |
| // Otherwise, default to http for localhost and https for everything else. |
| var scheme = |
| isLoopback(url.replaceAll(new RegExp(":.*"), "")) ? "http" : "https"; |
| return "$scheme://$url"; |
| } |
| |
| /// Returns the server URL for [description]. |
| Uri _serverFor(description) => |
| Uri.parse(source._parseDescription(description).last); |
| |
| /// Parses [id] into its server, package name, and version components, then |
| /// converts that to a Uri given [pattern]. |
| /// |
| /// Ensures the package name is properly URL encoded. |
| Uri _makeVersionUrl(PackageId id, |
| String pattern(String server, String package, String version)) { |
| var parsed = source._parseDescription(id.description); |
| var server = parsed.last; |
| var package = Uri.encodeComponent(parsed.first); |
| var version = Uri.encodeComponent(id.version.toString()); |
| return Uri.parse(pattern(server, package, version)); |
| } |
| } |
| |
| /// This is the modified hosted source used when pub get or upgrade are run |
| /// with "--offline". |
| /// |
| /// This uses the system cache to get the list of available packages and does |
| /// no network access. |
| class _OfflineHostedSource extends BoundHostedSource { |
| _OfflineHostedSource(HostedSource source, SystemCache systemCache) |
| : super(source, systemCache); |
| |
| /// Gets the list of all versions of [ref] that are in the system cache. |
| Future<List<PackageId>> doGetVersions(PackageRef ref) async { |
| var parsed = source._parseDescription(ref.description); |
| var server = parsed.last; |
| log.io("Finding versions of ${ref.name} in " |
| "$systemCacheRoot/${_urlToDirectory(server)}"); |
| |
| var dir = p.join(systemCacheRoot, _urlToDirectory(server)); |
| |
| var versions; |
| if (dirExists(dir)) { |
| versions = await listDir(dir) |
| .map((entry) { |
| var components = p.basename(entry).split("-"); |
| if (components.first != ref.name) return null; |
| return source.idFor( |
| ref.name, new Version.parse(components.skip(1).join("-")), |
| url: _serverFor(ref.description)); |
| }) |
| .where((id) => id != null) |
| .toList(); |
| } else { |
| versions = []; |
| } |
| |
| // If there are no versions in the cache, report a clearer error. |
| if (versions.isEmpty) { |
| throw new PackageNotFoundException( |
| "could not find package ${ref.name} in cache"); |
| } |
| |
| return versions; |
| } |
| |
| Future _download( |
| String server, String package, Version version, String destPath) { |
| // Since HostedSource is cached, this will only be called for uncached |
| // packages. |
| throw new UnsupportedError("Cannot download packages when offline."); |
| } |
| |
| Future<Pubspec> describeUncached(PackageId id) { |
| throw new PackageNotFoundException( |
| "${id.name} ${id.version} is not available in your system cache"); |
| } |
| } |