blob: 5f1def9f0f9f32b6bcc9db38dae9bb098e1e7a00 [file] [log] [blame]
// Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
// for details. All rights reserved. Use of this source code is governed by a
// BSD-style license that can be found in the LICENSE file.
// @dart=2.10
import 'dart:async';
import 'dart:io';
import 'package:path/path.dart' as p;
import 'package:pool/pool.dart';
import 'package:pub_semver/pub_semver.dart';
import '../git.dart' as git;
import '../io.dart';
import '../log.dart' as log;
import '../package.dart';
import '../package_name.dart';
import '../pubspec.dart';
import '../source.dart';
import '../system_cache.dart';
import '../utils.dart';
import 'cached.dart';
/// A package source that gets packages from Git repos.
class GitSource extends Source {
final name = 'git';
BoundGitSource bind(SystemCache systemCache) =>
BoundGitSource(this, systemCache);
/// Given a valid git package description, returns the URL of the repository
/// it pulls from.
/// If the url is relative, it will be returned relative to current working
/// directory.
String urlFromDescription(description) {
var url = description['url'];
if (description['relative'] == true) {
return p.url.relative(url, from: p.toUri(p.current).toString());
return url;
PackageRef parseRef(String name, description, {String containingPath}) {
dynamic url;
dynamic ref;
dynamic path;
if (description is String) {
url = description;
} else if (description is! Map) {
throw FormatException('The description must be a Git URL or a map '
"with a 'url' key.");
} else {
url = description['url'];
ref = description['ref'];
if (ref != null && ref is! String) {
throw FormatException("The 'ref' field of the description must be a "
path = description['path'];
return PackageRef(name, this, {
..._validatedUrl(url, containingPath),
'ref': ref ?? 'HEAD',
'path': _validatedPath(path),
PackageId parseId(String name, Version version, description,
{String containingPath}) {
if (description is! Map) {
throw FormatException("The description must be a map with a 'url' "
var ref = description['ref'];
if (ref != null && ref is! String) {
throw FormatException("The 'ref' field of the description must be a "
if (description['resolved-ref'] is! String) {
throw FormatException("The 'resolved-ref' field of the description "
'must be a string.');
return PackageId(name, this, version, {
..._validatedUrl(description['url'], containingPath),
'ref': ref ?? 'HEAD',
'resolved-ref': description['resolved-ref'],
'path': _validatedPath(description['path'])
/// Serializes path dependency's [description].
/// For the descriptions where `relative` attribute is `true`, tries to make
/// `url` relative to the specified [containingPath].
dynamic serializeDescription(String containingPath, description) {
final copy = Map.from(description);
if (description['relative'] == true) {
copy['url'] = p.url.relative(description['url'],
from: Uri.file(containingPath).toString());
return copy;
/// Throws a [FormatException] if [url] isn't a valid Git URL.
Map<String, Object> _validatedUrl(dynamic url, String containingDir) {
if (url is! String) {
throw FormatException("The 'url' field of the description must be a "
var relative = false;
// If the URL contains an @, it's probably an SSH hostname, which we don't
// know how to validate.
if (!url.contains('@')) {
// Otherwise, we use Dart's URL parser to validate the URL.
final parsed = Uri.parse(url);
if (!parsed.hasAbsolutePath) {
// Relative paths coming from pubspecs that are not on the local file
// system aren't allowed. This can happen if a hosted or git dependency
// has a git dependency.
if (containingDir == null) {
throw FormatException('"$url" is a relative path, but this '
'isn\'t a local pubspec.');
// A relative path is stored internally as absolute resolved relative to
// [containingPath].
relative = true;
url = Uri.file(p.absolute(containingDir)).resolveUri(parsed).toString();
return {'relative': relative, 'url': url};
/// Returns [path] normalized.
/// Throws a [FormatException] if [path] isn't a relative url or null.
String _validatedPath(dynamic path) {
path ??= '.';
if (path is! String) {
throw FormatException("The 'path' field of the description must be a "
// Use Dart's URL parser to validate the URL.
final parsed = Uri.parse(path);
if (parsed.isAbsolute) {
throw FormatException(
"The 'path' field of the description must be relative.");
if (!p.url.isWithin('.', path) && !p.url.equals('.', path)) {
throw FormatException(
"The 'path' field of the description must not reach outside the "
return p.url.normalize(parsed.toString());
/// If [description] has a resolved ref, print it out in short-form.
/// This helps distinguish different git commits with the same pubspec
/// version.
String formatDescription(description) {
if (description is Map && description.containsKey('resolved-ref')) {
var result = '${urlFromDescription(description)} at '
"${description['resolved-ref'].substring(0, 6)}";
if (description['path'] != '.') result += " in ${description["path"]}";
return result;
} else {
return super.formatDescription(description);
/// Two Git descriptions are equal if both their URLs and their refs are
/// equal.
bool descriptionsEqual(description1, description2) {
// TODO(nweiz): Do we really want to throw an error if you have two
// dependencies on some repo, one of which specifies a ref and one of which
// doesn't? If not, how do we handle that case in the version solver?
if (description1['url'] != description2['url']) return false;
if (description1['ref'] != description2['ref']) return false;
if (description1['path'] != description2['path']) return false;
if (description1.containsKey('resolved-ref') &&
description2.containsKey('resolved-ref')) {
return description1['resolved-ref'] == description2['resolved-ref'];
return true;
int hashDescription(description) {
// Don't include the resolved ref in the hash code because we ignore it in
// [descriptionsEqual] if only one description defines it.
return description['url'].hashCode ^
description['ref'].hashCode ^
/// The [BoundSource] for [GitSource].
class BoundGitSource extends CachedSource {
/// Limit the number of concurrent git operations to 1.
// TODO(sigurdm): Use RateLimitedScheduler.
final Pool _pool = Pool(1);
final GitSource source;
final SystemCache systemCache;
/// A map from revision cache locations to futures that will complete once
/// they're finished being cloned.
/// This lets us avoid race conditions when getting multiple different
/// packages from the same repository.
final _revisionCacheClones = <String, Future>{};
/// The paths to the canonical clones of repositories for which "git fetch"
/// has already been run during this run of pub.
final _updatedRepos = <String>{};
BoundGitSource(this.source, this.systemCache);
/// Given a Git repo that contains a pub package, gets the name of the pub
/// package.
Future<String> getPackageNameFromRepo(String repo) {
// Clone the repo to a temp directory.
return withTempDir((tempDir) async {
await _clone(repo, tempDir, shallow: true);
var pubspec = Pubspec.load(tempDir, systemCache.sources);
Future<List<PackageId>> doGetVersions(PackageRef ref, Duration maxAge) async {
return await _pool.withResource(() async {
await _ensureRepoCache(ref);
var path = _repoCachePath(ref);
var revision = await _firstRevision(path, ref.description['ref']);
var pubspec = await _describeUncached(
return [
PackageId(, source, pubspec.version, {
'url': ref.description['url'],
'relative': ref.description['relative'],
'ref': ref.description['ref'],
'resolved-ref': revision,
'path': ref.description['path']
/// Since we don't have an easy way to read from a remote Git repo, this
/// just installs [id] into the system cache, then describes it from there.
Future<Pubspec> describeUncached(PackageId id) {
return _pool.withResource(() => _describeUncached(
/// Like [describeUncached], but takes a separate [ref] and Git [revision]
/// rather than a single ID.
Future<Pubspec> _describeUncached(
PackageRef ref,
String revision,
String path,
String url,
) async {
await _ensureRevision(ref, revision);
var repoPath = _repoCachePath(ref);
// Normalize the path because Git treats "./" at the beginning of a path
// specially.
var pubspecPath = p.normalize(p.join(p.fromUri(path), 'pubspec.yaml'));
// Git doesn't recognize backslashes in paths, even on Windows.
if (Platform.isWindows) pubspecPath = pubspecPath.replaceAll('\\', '/');
List<String> lines;
try {
lines = await git
.run(['show', '$revision:$pubspecPath'], workingDir: repoPath);
} on git.GitException catch (_) {
fail('Could not find a file named "$pubspecPath" in '
'${source.urlFromDescription(ref.description)} $revision.');
return Pubspec.parse(
/// Clones a Git repo to the local filesystem.
/// The Git cache directory is a little idiosyncratic. At the top level, it
/// contains a directory for each commit of each repository, named `<package
/// name>-<commit hash>`. These are the canonical package directories that are
/// linked to from the `packages/` directory.
/// In addition, the Git system cache contains a subdirectory named `cache/`
/// which contains a directory for each separate repository URL, named
/// `<package name>-<url hash>`. These are used to check out the repository
/// itself; each of the commit-specific directories are clones of a directory
/// in `cache/`.
Future<Package> downloadToSystemCache(PackageId id) async {
return await _pool.withResource(() async {
var ref = id.toRef();
if (!git.isInstalled) {
fail("Cannot get ${} from Git (${ref.description['url']}).\n"
'Please ensure Git is correctly installed.');
ensureDir(p.join(systemCacheRoot, 'cache'));
await _ensureRevision(ref, id.description['resolved-ref']);
var revisionCachePath = _revisionCachePath(id);
await _revisionCacheClones.putIfAbsent(revisionCachePath, () async {
if (!entryExists(revisionCachePath)) {
await _clone(_repoCachePath(ref), revisionCachePath);
await _checkOut(revisionCachePath, id.description['resolved-ref']);
_writePackageList(revisionCachePath, [id.description['path']]);
} else {
_updatePackageList(revisionCachePath, id.description['path']);
return Package.load(,
p.join(revisionCachePath, p.fromUri(id.description['path'])),
/// Returns the path to the revision-specific cache of [id].
String getDirectoryInCache(PackageId id) =>
p.join(_revisionCachePath(id), id.description['path']);
List<Package> getCachedPackages() {
// TODO(keertip): Implement getCachedPackages().
throw UnimplementedError(
"The git source doesn't support listing its cached packages yet.");
/// Resets all cached packages back to the pristine state of the Git
/// repository at the revision they are pinned to.
Future<Iterable<RepairResult>> repairCachedPackages() async {
if (!dirExists(systemCacheRoot)) return [];
final result = <RepairResult>[];
var packages = listDir(systemCacheRoot)
.where((entry) => dirExists(p.join(entry, '.git')))
.expand((revisionCachePath) {
return _readPackageList(revisionCachePath).map((relative) {
// If we've already failed to load another package from this
// repository, ignore it.
if (!dirExists(revisionCachePath)) return null;
var packageDir = p.join(revisionCachePath, relative);
try {
return Package.load(null, packageDir, systemCache.sources);
} catch (error, stackTrace) {
log.error('Failed to load package', error, stackTrace);
var name = p.basename(revisionCachePath).split('-').first;
PackageId(name, source, Version.none, '???'),
success: false));
return null;
.where((package) => package != null)
// Note that there may be multiple packages with the same name and version
// (pinned to different commits). The sort order of those is unspecified.
for (var package in packages) {
// If we've already failed to repair another package in this repository,
// ignore it.
if (!dirExists(package.dir)) continue;
var id = PackageId(, source, package.version, null);
log.message('Resetting Git repository for '
'${log.bold(} ${package.version}...');
try {
// Remove all untracked files.
await git
.run(['clean', '-d', '--force', '-x'], workingDir: package.dir);
// Discard all changes to tracked files.
await['reset', '--hard', 'HEAD'], workingDir: package.dir);
result.add(RepairResult(id, success: true));
} on git.GitException catch (error, stackTrace) {
log.error('Failed to reset ${log.bold(} '
'${package.version}. Error:\n$error');
result.add(RepairResult(id, success: false));
// Delete the revision cache path, not the subdirectory that contains the package.
return result;
/// Ensures that the canonical clone of the repository referred to by [ref]
/// contains the given Git [revision].
Future _ensureRevision(PackageRef ref, String revision) async {
var path = _repoCachePath(ref);
if (_updatedRepos.contains(path)) return;
await _deleteGitRepoIfInvalid(path);
if (!entryExists(path)) await _createRepoCache(ref);
// Try to list the revision. If it doesn't exist, git will fail and we'll
// know we have to update the repository.
try {
await _firstRevision(path, revision);
} on git.GitException catch (_) {
await _updateRepoCache(ref);
/// Ensures that the canonical clone of the repository referred to by [ref]
/// exists and is up-to-date.
Future _ensureRepoCache(PackageRef ref) async {
var path = _repoCachePath(ref);
if (_updatedRepos.contains(path)) return;
await _deleteGitRepoIfInvalid(path);
if (!entryExists(path)) {
await _createRepoCache(ref);
} else {
await _updateRepoCache(ref);
/// Creates the canonical clone of the repository referred to by [ref].
/// This assumes that the canonical clone doesn't yet exist.
Future _createRepoCache(PackageRef ref) async {
var path = _repoCachePath(ref);
try {
await _clone(ref.description['url'], path, mirror: true);
} catch (_) {
await _deleteGitRepoIfInvalid(path);
/// Runs "git fetch" in the canonical clone of the repository referred to by
/// [ref].
/// This assumes that the canonical clone already exists.
Future _updateRepoCache(PackageRef ref) async {
var path = _repoCachePath(ref);
if (_updatedRepos.contains(path)) return Future.value();
await['fetch'], workingDir: path);
/// Clean-up [dirPath] if it's an invalid git repository.
/// The git clones in the `PUB_CACHE` folder should never be invalid. But this
/// can happen if the clone operation failed in some way, and the program did
/// not exit gracefully, leaving the cache git clone in a dirty state.
Future<void> _deleteGitRepoIfInvalid(String dirPath) async {
if (!dirExists(dirPath)) {
var isValid = true;
try {
final result = await
['rev-parse', '--is-inside-git-dir'],
workingDir: dirPath,
if (result?.join('\n') != 'true') {
isValid = false;
} on git.GitException {
isValid = false;
// If [dirPath] is not a valid git repository we remove it.
if (!isValid) {
/// Updates the package list file in [revisionCachePath] to include [path], if
/// necessary.
void _updatePackageList(String revisionCachePath, String path) {
var packages = _readPackageList(revisionCachePath);
if (packages.contains(path)) return;
_writePackageList(revisionCachePath, packages..add(path));
/// Returns the list of packages in [revisionCachePath].
List<String> _readPackageList(String revisionCachePath) {
var path = _packageListPath(revisionCachePath);
// If there's no package list file, this cache was created by an older
// version of pub where pubspecs were only allowed at the root of the
// repository.
if (!fileExists(path)) return ['.'];
return readTextFile(path).split('\n');
/// Writes a package list indicating that [packages] exist in
/// [revisionCachePath].
void _writePackageList(String revisionCachePath, List<String> packages) {
writeTextFile(_packageListPath(revisionCachePath), packages.join('\n'));
/// The path in a revision cache repository in which we keep a list of the
/// packages in the repository.
String _packageListPath(String revisionCachePath) =>
p.join(revisionCachePath, '.git/pub-packages');
/// Runs "git rev-list" on [reference] in [path] and returns the first result.
/// This assumes that the canonical clone already exists.
Future<String> _firstRevision(String path, String reference) async {
var lines = await git
.run(['rev-list', '--max-count=1', reference], workingDir: path);
return lines.first;
/// Clones the repo at the URI [from] to the path [to] on the local
/// filesystem.
/// If [mirror] is true, creates a bare, mirrored clone. This doesn't check
/// out the working tree, but instead makes the repository a local mirror of
/// the remote repository. See the manpage for `git clone` for more
/// information.
/// If [shallow] is true, creates a shallow clone that contains no history
/// for the repository.
Future _clone(
String from,
String to, {
bool mirror = false,
bool shallow = false,
}) {
return Future.sync(() {
// Git on Windows does not seem to automatically create the destination
// directory.
var args = [
if (mirror) '--mirror',
if (shallow) ...['--depth', '1'],
}).then((result) => null);
/// Checks out the reference [ref] in [repoPath].
Future _checkOut(String repoPath, String ref) {
return git
.run(['checkout', ref], workingDir: repoPath).then((result) => null);
String _revisionCachePath(PackageId id) => p.join(
systemCacheRoot, "${_repoName(id)}-${id.description['resolved-ref']}");
/// Returns the path to the canonical clone of the repository referred to by
/// [id] (the one in `<system cache>/git/cache`).
String _repoCachePath(PackageRef ref) {
var repoCacheName = '${_repoName(ref)}-${sha1(ref.description['url'])}';
return p.join(systemCacheRoot, 'cache', repoCacheName);
/// Returns a short, human-readable name for the repository URL in [packageName].
/// This name is not guaranteed to be unique.
String _repoName(PackageName packageName) {
var name = p.url.basename(packageName.description['url']);
if (name.endsWith('.git')) {
name = name.substring(0, name.length - '.git'.length);
name = name.replaceAll(RegExp('[^a-zA-Z0-9._-]'), '_');
// Shorten name to 50 chars for sanity.
if (name.length > 50) {
name = name.substring(0, 50);
return name;