| // Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file |
| // for details. All rights reserved. Use of this source code is governed by a |
| // BSD-style license that can be found in the LICENSE file. |
| |
| import 'dart:async'; |
| import 'dart:io'; |
| |
| import 'package:collection/collection.dart' show IterableNullableExtension; |
| import 'package:path/path.dart' as p; |
| import 'package:pool/pool.dart'; |
| import 'package:pub_semver/pub_semver.dart'; |
| |
| import '../git.dart' as git; |
| import '../io.dart'; |
| import '../log.dart' as log; |
| import '../package.dart'; |
| import '../package_name.dart'; |
| import '../pubspec.dart'; |
| import '../source.dart'; |
| import '../system_cache.dart'; |
| import '../utils.dart'; |
| import 'cached.dart'; |
| |
| /// A package source that gets packages from Git repos. |
| class GitSource extends Source { |
| @override |
| final name = 'git'; |
| |
| @override |
| BoundGitSource bind(SystemCache systemCache) => |
| BoundGitSource(this, systemCache); |
| |
| /// Given a valid git package description, returns the URL of the repository |
| /// it pulls from. |
| /// If the url is relative, it will be returned relative to current working |
| /// directory. |
| String urlFromDescription(description) { |
| var url = description['url']; |
| if (description['relative'] == true) { |
| return p.url.relative(url, from: p.toUri(p.current).toString()); |
| } |
| return url; |
| } |
| |
| @override |
| PackageRef parseRef(String name, description, {String? containingPath}) { |
| dynamic url; |
| dynamic ref; |
| dynamic path; |
| if (description is String) { |
| url = description; |
| } else if (description is! Map) { |
| throw FormatException('The description must be a Git URL or a map ' |
| "with a 'url' key."); |
| } else { |
| url = description['url']; |
| |
| ref = description['ref']; |
| if (ref != null && ref is! String) { |
| throw FormatException("The 'ref' field of the description must be a " |
| 'string.'); |
| } |
| |
| path = description['path']; |
| } |
| return PackageRef(name, this, { |
| ..._validatedUrl(url, containingPath), |
| 'ref': ref ?? 'HEAD', |
| 'path': _validatedPath(path), |
| }); |
| } |
| |
| @override |
| PackageId parseId(String name, Version version, description, |
| {String? containingPath}) { |
| if (description is! Map) { |
| throw FormatException("The description must be a map with a 'url' " |
| 'key.'); |
| } |
| |
| var ref = description['ref']; |
| if (ref != null && ref is! String) { |
| throw FormatException("The 'ref' field of the description must be a " |
| 'string.'); |
| } |
| |
| if (description['resolved-ref'] is! String) { |
| throw FormatException("The 'resolved-ref' field of the description " |
| 'must be a string.'); |
| } |
| |
| return PackageId(name, this, version, { |
| ..._validatedUrl(description['url'], containingPath), |
| 'ref': ref ?? 'HEAD', |
| 'resolved-ref': description['resolved-ref'], |
| 'path': _validatedPath(description['path']) |
| }); |
| } |
| |
| /// Serializes path dependency's [description]. |
| /// |
| /// For the descriptions where `relative` attribute is `true`, tries to make |
| /// `url` relative to the specified [containingPath]. |
| @override |
| dynamic serializeDescription(String containingPath, description) { |
| final copy = Map.from(description); |
| copy.remove('relative'); |
| if (description['relative'] == true) { |
| copy['url'] = p.url.relative(description['url'], |
| from: Uri.file(containingPath).toString()); |
| } |
| return copy; |
| } |
| |
| /// Throws a [FormatException] if [url] isn't a valid Git URL. |
| Map<String, Object> _validatedUrl(dynamic url, String? containingDir) { |
| if (url is! String) { |
| throw FormatException("The 'url' field of the description must be a " |
| 'string.'); |
| } |
| var relative = false; |
| // If the URL contains an @, it's probably an SSH hostname, which we don't |
| // know how to validate. |
| if (!url.contains('@')) { |
| // Otherwise, we use Dart's URL parser to validate the URL. |
| final parsed = Uri.parse(url); |
| if (!parsed.hasAbsolutePath) { |
| // Relative paths coming from pubspecs that are not on the local file |
| // system aren't allowed. This can happen if a hosted or git dependency |
| // has a git dependency. |
| if (containingDir == null) { |
| throw FormatException('"$url" is a relative path, but this ' |
| 'isn\'t a local pubspec.'); |
| } |
| // A relative path is stored internally as absolute resolved relative to |
| // [containingPath]. |
| relative = true; |
| url = Uri.file(p.absolute(containingDir)).resolveUri(parsed).toString(); |
| } |
| } |
| return {'relative': relative, 'url': url}; |
| } |
| |
| /// Returns [path] normalized. |
| /// |
| /// Throws a [FormatException] if [path] isn't a relative url or null. |
| String _validatedPath(dynamic path) { |
| path ??= '.'; |
| if (path is! String) { |
| throw FormatException("The 'path' field of the description must be a " |
| 'string.'); |
| } |
| |
| // Use Dart's URL parser to validate the URL. |
| final parsed = Uri.parse(path); |
| if (parsed.isAbsolute) { |
| throw FormatException( |
| "The 'path' field of the description must be relative."); |
| } |
| if (!p.url.isWithin('.', path) && !p.url.equals('.', path)) { |
| throw FormatException( |
| "The 'path' field of the description must not reach outside the " |
| 'repository.'); |
| } |
| return p.url.normalize(parsed.toString()); |
| } |
| |
| /// If [description] has a resolved ref, print it out in short-form. |
| /// |
| /// This helps distinguish different git commits with the same pubspec |
| /// version. |
| @override |
| String formatDescription(description) { |
| if (description is Map && description.containsKey('resolved-ref')) { |
| var result = '${urlFromDescription(description)} at ' |
| "${description['resolved-ref'].substring(0, 6)}"; |
| if (description['path'] != '.') result += " in ${description["path"]}"; |
| return result; |
| } else { |
| return super.formatDescription(description); |
| } |
| } |
| |
| /// Two Git descriptions are equal if both their URLs and their refs are |
| /// equal. |
| @override |
| bool descriptionsEqual(description1, description2) { |
| // TODO(nweiz): Do we really want to throw an error if you have two |
| // dependencies on some repo, one of which specifies a ref and one of which |
| // doesn't? If not, how do we handle that case in the version solver? |
| if (description1['url'] != description2['url']) return false; |
| if (description1['ref'] != description2['ref']) return false; |
| if (description1['path'] != description2['path']) return false; |
| |
| if (description1.containsKey('resolved-ref') && |
| description2.containsKey('resolved-ref')) { |
| return description1['resolved-ref'] == description2['resolved-ref']; |
| } |
| |
| return true; |
| } |
| |
| @override |
| int hashDescription(description) { |
| // Don't include the resolved ref in the hash code because we ignore it in |
| // [descriptionsEqual] if only one description defines it. |
| return description['url'].hashCode ^ |
| description['ref'].hashCode ^ |
| description['path'].hashCode; |
| } |
| } |
| |
| /// The [BoundSource] for [GitSource]. |
| class BoundGitSource extends CachedSource { |
| /// Limit the number of concurrent git operations to 1. |
| // TODO(sigurdm): Use RateLimitedScheduler. |
| final Pool _pool = Pool(1); |
| |
| @override |
| final GitSource source; |
| |
| @override |
| final SystemCache systemCache; |
| |
| /// A map from revision cache locations to futures that will complete once |
| /// they're finished being cloned. |
| /// |
| /// This lets us avoid race conditions when getting multiple different |
| /// packages from the same repository. |
| final _revisionCacheClones = <String, Future>{}; |
| |
| /// The paths to the canonical clones of repositories for which "git fetch" |
| /// has already been run during this run of pub. |
| final _updatedRepos = <String>{}; |
| |
| BoundGitSource(this.source, this.systemCache); |
| |
| /// Given a Git repo that contains a pub package, gets the name of the pub |
| /// package. |
| Future<String> getPackageNameFromRepo(String repo) { |
| // Clone the repo to a temp directory. |
| return withTempDir((tempDir) async { |
| await _clone(repo, tempDir, shallow: true); |
| var pubspec = Pubspec.load(tempDir, systemCache.sources); |
| return pubspec.name; |
| }); |
| } |
| |
| @override |
| Future<List<PackageId>> doGetVersions( |
| PackageRef ref, Duration? maxAge) async { |
| return await _pool.withResource(() async { |
| await _ensureRepoCache(ref); |
| var path = _repoCachePath(ref); |
| var revision = await _firstRevision(path, ref.description['ref']); |
| var pubspec = await _describeUncached( |
| ref, |
| revision, |
| ref.description['path'], |
| source.urlFromDescription(ref.description), |
| ); |
| |
| return [ |
| PackageId(ref.name, source, pubspec.version, { |
| 'url': ref.description['url'], |
| 'relative': ref.description['relative'], |
| 'ref': ref.description['ref'], |
| 'resolved-ref': revision, |
| 'path': ref.description['path'] |
| }) |
| ]; |
| }); |
| } |
| |
| /// Since we don't have an easy way to read from a remote Git repo, this |
| /// just installs [id] into the system cache, then describes it from there. |
| @override |
| Future<Pubspec> describeUncached(PackageId id) { |
| return _pool.withResource(() => _describeUncached( |
| id.toRef(), |
| id.description['resolved-ref'], |
| id.description['path'], |
| source.urlFromDescription(id.description), |
| )); |
| } |
| |
| /// Like [describeUncached], but takes a separate [ref] and Git [revision] |
| /// rather than a single ID. |
| Future<Pubspec> _describeUncached( |
| PackageRef ref, |
| String revision, |
| String path, |
| String url, |
| ) async { |
| await _ensureRevision(ref, revision); |
| var repoPath = _repoCachePath(ref); |
| |
| // Normalize the path because Git treats "./" at the beginning of a path |
| // specially. |
| var pubspecPath = p.normalize(p.join(p.fromUri(path), 'pubspec.yaml')); |
| |
| // Git doesn't recognize backslashes in paths, even on Windows. |
| if (Platform.isWindows) pubspecPath = pubspecPath.replaceAll('\\', '/'); |
| late List<String> lines; |
| try { |
| lines = await git |
| .run(['show', '$revision:$pubspecPath'], workingDir: repoPath); |
| } on git.GitException catch (_) { |
| fail('Could not find a file named "$pubspecPath" in ' |
| '${source.urlFromDescription(ref.description)} $revision.'); |
| } |
| |
| return Pubspec.parse( |
| lines.join('\n'), |
| systemCache.sources, |
| expectedName: ref.name, |
| ); |
| } |
| |
| /// Clones a Git repo to the local filesystem. |
| /// |
| /// The Git cache directory is a little idiosyncratic. At the top level, it |
| /// contains a directory for each commit of each repository, named `<package |
| /// name>-<commit hash>`. These are the canonical package directories that are |
| /// linked to from the `packages/` directory. |
| /// |
| /// In addition, the Git system cache contains a subdirectory named `cache/` |
| /// which contains a directory for each separate repository URL, named |
| /// `<package name>-<url hash>`. These are used to check out the repository |
| /// itself; each of the commit-specific directories are clones of a directory |
| /// in `cache/`. |
| @override |
| Future<Package> downloadToSystemCache(PackageId id) async { |
| return await _pool.withResource(() async { |
| var ref = id.toRef(); |
| if (!git.isInstalled) { |
| fail("Cannot get ${id.name} from Git (${ref.description['url']}).\n" |
| 'Please ensure Git is correctly installed.'); |
| } |
| |
| ensureDir(p.join(systemCacheRoot, 'cache')); |
| await _ensureRevision(ref, id.description['resolved-ref']); |
| |
| var revisionCachePath = _revisionCachePath(id); |
| await _revisionCacheClones.putIfAbsent(revisionCachePath, () async { |
| if (!entryExists(revisionCachePath)) { |
| await _clone(_repoCachePath(ref), revisionCachePath); |
| await _checkOut(revisionCachePath, id.description['resolved-ref']); |
| _writePackageList(revisionCachePath, [id.description['path']]); |
| } else { |
| _updatePackageList(revisionCachePath, id.description['path']); |
| } |
| }); |
| |
| return Package.load( |
| id.name, |
| p.join(revisionCachePath, p.fromUri(id.description['path'])), |
| systemCache.sources); |
| }); |
| } |
| |
| /// Returns the path to the revision-specific cache of [id]. |
| @override |
| String getDirectoryInCache(PackageId? id) => |
| p.join(_revisionCachePath(id!), id.description['path']); |
| |
| @override |
| List<Package> getCachedPackages() { |
| // TODO(keertip): Implement getCachedPackages(). |
| throw UnimplementedError( |
| "The git source doesn't support listing its cached packages yet."); |
| } |
| |
| /// Resets all cached packages back to the pristine state of the Git |
| /// repository at the revision they are pinned to. |
| @override |
| Future<Iterable<RepairResult>> repairCachedPackages() async { |
| if (!dirExists(systemCacheRoot)) return []; |
| |
| final result = <RepairResult>[]; |
| |
| var packages = listDir(systemCacheRoot) |
| .where((entry) => dirExists(p.join(entry, '.git'))) |
| .expand((revisionCachePath) { |
| return _readPackageList(revisionCachePath).map((relative) { |
| // If we've already failed to load another package from this |
| // repository, ignore it. |
| if (!dirExists(revisionCachePath)) return null; |
| |
| var packageDir = p.join(revisionCachePath, relative); |
| try { |
| return Package.load(null, packageDir, systemCache.sources); |
| } catch (error, stackTrace) { |
| log.error('Failed to load package', error, stackTrace); |
| var name = p.basename(revisionCachePath).split('-').first; |
| result.add(RepairResult( |
| PackageId(name, source, Version.none, '???'), |
| success: false)); |
| tryDeleteEntry(revisionCachePath); |
| return null; |
| } |
| }); |
| }) |
| .whereNotNull() |
| .toList(); |
| |
| // Note that there may be multiple packages with the same name and version |
| // (pinned to different commits). The sort order of those is unspecified. |
| packages.sort(Package.orderByNameAndVersion); |
| |
| for (var package in packages) { |
| // If we've already failed to repair another package in this repository, |
| // ignore it. |
| if (!dirExists(package.dir)) continue; |
| |
| var id = PackageId(package.name, source, package.version, null); |
| |
| log.message('Resetting Git repository for ' |
| '${log.bold(package.name)} ${package.version}...'); |
| |
| try { |
| // Remove all untracked files. |
| await git |
| .run(['clean', '-d', '--force', '-x'], workingDir: package.dir); |
| |
| // Discard all changes to tracked files. |
| await git.run(['reset', '--hard', 'HEAD'], workingDir: package.dir); |
| |
| result.add(RepairResult(id, success: true)); |
| } on git.GitException catch (error, stackTrace) { |
| log.error('Failed to reset ${log.bold(package.name)} ' |
| '${package.version}. Error:\n$error'); |
| log.fine(stackTrace); |
| result.add(RepairResult(id, success: false)); |
| |
| // Delete the revision cache path, not the subdirectory that contains the package. |
| tryDeleteEntry(getDirectoryInCache(id)); |
| } |
| } |
| |
| return result; |
| } |
| |
| /// Ensures that the canonical clone of the repository referred to by [ref] |
| /// contains the given Git [revision]. |
| Future _ensureRevision(PackageRef ref, String revision) async { |
| var path = _repoCachePath(ref); |
| if (_updatedRepos.contains(path)) return; |
| |
| await _deleteGitRepoIfInvalid(path); |
| |
| if (!entryExists(path)) await _createRepoCache(ref); |
| |
| // Try to list the revision. If it doesn't exist, git will fail and we'll |
| // know we have to update the repository. |
| try { |
| await _firstRevision(path, revision); |
| } on git.GitException catch (_) { |
| await _updateRepoCache(ref); |
| } |
| } |
| |
| /// Ensures that the canonical clone of the repository referred to by [ref] |
| /// exists and is up-to-date. |
| Future _ensureRepoCache(PackageRef ref) async { |
| var path = _repoCachePath(ref); |
| if (_updatedRepos.contains(path)) return; |
| |
| await _deleteGitRepoIfInvalid(path); |
| |
| if (!entryExists(path)) { |
| await _createRepoCache(ref); |
| } else { |
| await _updateRepoCache(ref); |
| } |
| } |
| |
| /// Creates the canonical clone of the repository referred to by [ref]. |
| /// |
| /// This assumes that the canonical clone doesn't yet exist. |
| Future _createRepoCache(PackageRef ref) async { |
| var path = _repoCachePath(ref); |
| assert(!_updatedRepos.contains(path)); |
| try { |
| await _clone(ref.description['url'], path, mirror: true); |
| } catch (_) { |
| await _deleteGitRepoIfInvalid(path); |
| rethrow; |
| } |
| _updatedRepos.add(path); |
| } |
| |
| /// Runs "git fetch" in the canonical clone of the repository referred to by |
| /// [ref]. |
| /// |
| /// This assumes that the canonical clone already exists. |
| Future _updateRepoCache(PackageRef ref) async { |
| var path = _repoCachePath(ref); |
| if (_updatedRepos.contains(path)) return Future.value(); |
| await git.run(['fetch'], workingDir: path); |
| _updatedRepos.add(path); |
| } |
| |
| /// Clean-up [dirPath] if it's an invalid git repository. |
| /// |
| /// The git clones in the `PUB_CACHE` folder should never be invalid. But this |
| /// can happen if the clone operation failed in some way, and the program did |
| /// not exit gracefully, leaving the cache git clone in a dirty state. |
| Future<void> _deleteGitRepoIfInvalid(String dirPath) async { |
| if (!dirExists(dirPath)) { |
| return; |
| } |
| var isValid = true; |
| try { |
| final result = await git.run( |
| ['rev-parse', '--is-inside-git-dir'], |
| workingDir: dirPath, |
| ); |
| if (result.join('\n') != 'true') { |
| isValid = false; |
| } |
| } on git.GitException { |
| isValid = false; |
| } |
| // If [dirPath] is not a valid git repository we remove it. |
| if (!isValid) { |
| deleteEntry(dirPath); |
| } |
| } |
| |
| /// Updates the package list file in [revisionCachePath] to include [path], if |
| /// necessary. |
| void _updatePackageList(String revisionCachePath, String path) { |
| var packages = _readPackageList(revisionCachePath); |
| if (packages.contains(path)) return; |
| |
| _writePackageList(revisionCachePath, packages..add(path)); |
| } |
| |
| /// Returns the list of packages in [revisionCachePath]. |
| List<String> _readPackageList(String revisionCachePath) { |
| var path = _packageListPath(revisionCachePath); |
| |
| // If there's no package list file, this cache was created by an older |
| // version of pub where pubspecs were only allowed at the root of the |
| // repository. |
| if (!fileExists(path)) return ['.']; |
| return readTextFile(path).split('\n'); |
| } |
| |
| /// Writes a package list indicating that [packages] exist in |
| /// [revisionCachePath]. |
| void _writePackageList(String revisionCachePath, List<String> packages) { |
| writeTextFile(_packageListPath(revisionCachePath), packages.join('\n')); |
| } |
| |
| /// The path in a revision cache repository in which we keep a list of the |
| /// packages in the repository. |
| String _packageListPath(String revisionCachePath) => |
| p.join(revisionCachePath, '.git/pub-packages'); |
| |
| /// Runs "git rev-list" on [reference] in [path] and returns the first result. |
| /// |
| /// This assumes that the canonical clone already exists. |
| Future<String> _firstRevision(String path, String reference) async { |
| var lines = await git |
| .run(['rev-list', '--max-count=1', reference], workingDir: path); |
| return lines.first; |
| } |
| |
| /// Clones the repo at the URI [from] to the path [to] on the local |
| /// filesystem. |
| /// |
| /// If [mirror] is true, creates a bare, mirrored clone. This doesn't check |
| /// out the working tree, but instead makes the repository a local mirror of |
| /// the remote repository. See the manpage for `git clone` for more |
| /// information. |
| /// |
| /// If [shallow] is true, creates a shallow clone that contains no history |
| /// for the repository. |
| Future _clone( |
| String from, |
| String to, { |
| bool mirror = false, |
| bool shallow = false, |
| }) { |
| return Future.sync(() { |
| // Git on Windows does not seem to automatically create the destination |
| // directory. |
| ensureDir(to); |
| var args = [ |
| 'clone', |
| if (mirror) '--mirror', |
| if (shallow) ...['--depth', '1'], |
| from, |
| to |
| ]; |
| |
| return git.run(args); |
| }).then((result) => null); |
| } |
| |
| /// Checks out the reference [ref] in [repoPath]. |
| Future _checkOut(String repoPath, String ref) { |
| return git |
| .run(['checkout', ref], workingDir: repoPath).then((result) => null); |
| } |
| |
| String _revisionCachePath(PackageId id) => p.join( |
| systemCacheRoot, "${_repoName(id)}-${id.description['resolved-ref']}"); |
| |
| /// Returns the path to the canonical clone of the repository referred to by |
| /// [id] (the one in `<system cache>/git/cache`). |
| String _repoCachePath(PackageRef ref) { |
| var repoCacheName = '${_repoName(ref)}-${sha1(ref.description['url'])}'; |
| return p.join(systemCacheRoot, 'cache', repoCacheName); |
| } |
| |
| /// Returns a short, human-readable name for the repository URL in [packageName]. |
| /// |
| /// This name is not guaranteed to be unique. |
| String _repoName(PackageName packageName) { |
| var name = p.url.basename(packageName.description['url']); |
| if (name.endsWith('.git')) { |
| name = name.substring(0, name.length - '.git'.length); |
| } |
| name = name.replaceAll(RegExp('[^a-zA-Z0-9._-]'), '_'); |
| // Shorten name to 50 chars for sanity. |
| if (name.length > 50) { |
| name = name.substring(0, 50); |
| } |
| return name; |
| } |
| } |