New command `dart pub cache preload` (#3636)
diff --git a/lib/src/command/cache.dart b/lib/src/command/cache.dart
index 12c4fbf..43e8c5a 100644
--- a/lib/src/command/cache.dart
+++ b/lib/src/command/cache.dart
@@ -6,6 +6,7 @@
import 'cache_add.dart';
import 'cache_clean.dart';
import 'cache_list.dart';
+import 'cache_preload.dart';
import 'cache_repair.dart';
/// Handles the `cache` pub command.
@@ -22,5 +23,8 @@
addSubcommand(CacheListCommand());
addSubcommand(CacheCleanCommand());
addSubcommand(CacheRepairCommand());
+ addSubcommand(
+ CachePreloadCommand(),
+ );
}
}
diff --git a/lib/src/command/cache_preload.dart b/lib/src/command/cache_preload.dart
new file mode 100644
index 0000000..54b7867
--- /dev/null
+++ b/lib/src/command/cache_preload.dart
@@ -0,0 +1,49 @@
+// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:async';
+
+import '../command.dart';
+import '../io.dart';
+import '../log.dart' as log;
+import '../source/hosted.dart';
+import '../utils.dart';
+
+/// Handles the `cache preload` pub command.
+class CachePreloadCommand extends PubCommand {
+ @override
+ String get name => 'preload';
+ @override
+ String get description => 'Install packages from a .tar.gz archive.';
+ @override
+ String get argumentsDescription => '<package1.tar.gz> ...';
+ @override
+ String get docUrl => 'https://dart.dev/tools/pub/cmd/pub-cache';
+
+ /// The `cache preload` command is hidden by default, because it's really only intended for
+ /// `flutter` to use when pre-loading `PUB_CACHE` after being installed from `zip` archive.
+ @override
+ bool get hidden => true;
+
+ @override
+ Future<void> runProtected() async {
+ // Make sure there is a package.
+ if (argResults.rest.isEmpty) {
+ usageException('No package to preload given.');
+ }
+
+ for (String packagePath in argResults.rest) {
+ if (!fileExists(packagePath)) {
+ fail('Could not find file $packagePath.');
+ }
+ }
+ for (String archivePath in argResults.rest) {
+ final id = await cache.hosted.preloadPackage(archivePath, cache);
+ final url = (id.description.description as HostedDescription).url;
+
+ final fromPart = HostedSource.isFromPubDev(id) ? '' : ' from $url';
+ log.message('Installed $archivePath in cache as $id$fromPart.');
+ }
+ }
+}
diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart
index 4e8a939..5922e3c 100644
--- a/lib/src/source/hosted.dart
+++ b/lib/src/source/hosted.dart
@@ -129,6 +129,11 @@
static bool isPubDevUrl(String url) {
final origin = Uri.parse(url).origin;
+ // Allow the defaultHostedUrl to be overriden when running from tests
+ if (runningFromTest &&
+ io.Platform.environment['_PUB_TEST_DEFAULT_HOSTED_URL'] != null) {
+ return origin == io.Platform.environment['_PUB_TEST_DEFAULT_HOSTED_URL'];
+ }
return origin == pubDevUrl || origin == pubDartlangUrl;
}
@@ -994,7 +999,7 @@
versions.firstWhereOrNull((i) => i.version == id.version);
final packageName = id.name;
final version = id.version;
- late Uint8List contentHash;
+ late final Uint8List contentHash;
if (versionInfo == null) {
throw PackageNotFoundException(
'Package $packageName has no version $version');
@@ -1032,13 +1037,8 @@
See $contentHashesDocumentationUrl.
''');
}
- final path = hashPath(id, cache);
- ensureDir(p.dirname(path));
- writeTextFile(
- path,
- hexEncode(actualHash.bytes),
- );
contentHash = Uint8List.fromList(actualHash.bytes);
+ writeHash(id, cache, contentHash);
}
// It is important that we do not compare against id.description.sha256,
@@ -1086,9 +1086,14 @@
);
var tempDir = cache.createTempDir();
- await extractTarGz(readBinaryFileAsStream(archivePath), tempDir);
+ try {
+ await extractTarGz(readBinaryFileAsStream(archivePath), tempDir);
- ensureDir(p.dirname(destPath));
+ ensureDir(p.dirname(destPath));
+ } catch (e) {
+ deleteEntry(tempDir);
+ rethrow;
+ }
// Now that the get has succeeded, move it to the real location in the
// cache.
//
@@ -1100,6 +1105,84 @@
});
}
+ /// Writes the contenthash for [id] in the cache.
+ void writeHash(PackageId id, SystemCache cache, List<int> bytes) {
+ final path = hashPath(id, cache);
+ ensureDir(p.dirname(path));
+ writeTextFile(
+ path,
+ hexEncode(bytes),
+ );
+ }
+
+ /// Installs a tar.gz file in [archivePath] as if it was downloaded from a
+ /// package repository.
+ ///
+ /// The name, version and repository are decided from the pubspec.yaml that
+ /// must be present in the archive.
+ Future<PackageId> preloadPackage(
+ String archivePath, SystemCache cache) async {
+ // Extract to a temp-folder and do atomic rename to preserve the integrity
+ // of the cache.
+ late final Uint8List contentHash;
+
+ var tempDir = cache.createTempDir();
+ final PackageId id;
+ try {
+ try {
+ // We read the file twice, once to compute the hash, and once to extract
+ // the archive.
+ //
+ // It would be desirable to read the file only once, but the tar
+ // extraction closes the stream early making things tricky to get right.
+ contentHash = Uint8List.fromList(
+ (await sha256.bind(readBinaryFileAsStream(archivePath)).first)
+ .bytes);
+ await extractTarGz(readBinaryFileAsStream(archivePath), tempDir);
+ } on FormatException catch (e) {
+ dataError('Failed to extract `$archivePath`: ${e.message}.');
+ }
+ if (!fileExists(p.join(tempDir, 'pubspec.yaml'))) {
+ fail(
+ 'Found no `pubspec.yaml` in $archivePath. Is it a valid pub package archive?');
+ }
+ final Pubspec pubspec;
+ try {
+ pubspec = Pubspec.load(tempDir, cache.sources);
+ final errors = pubspec.allErrors;
+ if (errors.isNotEmpty) {
+ throw errors.first;
+ }
+ } on Exception catch (e) {
+ fail('Failed to load `pubspec.yaml` from `$archivePath`: $e.');
+ }
+ // Reconstruct the PackageId from the extracted pubspec.yaml.
+ id = PackageId(
+ pubspec.name,
+ pubspec.version,
+ ResolvedHostedDescription(
+ HostedDescription(
+ pubspec.name,
+ validateAndNormalizeHostedUrl(cache.hosted.defaultUrl).toString(),
+ ),
+ sha256: contentHash,
+ ),
+ );
+ } catch (e) {
+ deleteEntry(tempDir);
+ rethrow;
+ }
+ final packageDir = getDirectoryInCache(id, cache);
+ if (dirExists(packageDir)) {
+ log.fine(
+ 'Cache entry for ${id.name}-${id.version} already exists. Replacing.');
+ deleteEntry(packageDir);
+ }
+ tryRenameDir(tempDir, packageDir);
+ writeHash(id, cache, contentHash);
+ return id;
+ }
+
/// When an error occurs trying to read something about [package] from [hostedUrl],
/// this tries to translate into a more user friendly error message.
///
diff --git a/test/cache/preload_test.dart b/test/cache/preload_test.dart
new file mode 100644
index 0000000..3bf5f64
--- /dev/null
+++ b/test/cache/preload_test.dart
@@ -0,0 +1,180 @@
+// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:http/http.dart';
+import 'package:path/path.dart' as p;
+import 'package:pub/src/exit_codes.dart';
+import 'package:test/test.dart';
+
+import '../descriptor.dart' as d;
+import '../descriptor.dart';
+import '../test_pub.dart';
+
+void main() {
+ test('adds correct entries to cache and stores the content-hash', () async {
+ final server = await servePackages();
+ server.serve('foo', '1.0.0');
+ server.serve('foo', '2.0.0');
+
+ await appDir({'foo': '^2.0.0'}).create();
+ // Do a `pub get` here to create a lock file in order to validate we later can
+ // `pub get --offline` with packages installed by `preload`.
+ await pubGet();
+
+ await runPub(args: ['cache', 'clean', '-f']);
+
+ final archivePath1 = p.join(sandbox, 'foo-1.0.0-archive.tar.gz');
+ final archivePath2 = p.join(sandbox, 'foo-2.0.0-archive.tar.gz');
+
+ File(archivePath1).writeAsBytesSync(await readBytes(
+ Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz')));
+ File(archivePath2).writeAsBytesSync(await readBytes(
+ Uri.parse(server.url).resolve('packages/foo/versions/2.0.0.tar.gz')));
+ await runPub(
+ args: ['cache', 'preload', archivePath1, archivePath2],
+ environment: {'_PUB_TEST_DEFAULT_HOSTED_URL': server.url},
+ output: allOf(
+ [
+ contains('Installed $archivePath1 in cache as foo 1.0.0.'),
+ contains('Installed $archivePath2 in cache as foo 2.0.0.'),
+ ],
+ ),
+ );
+ await d.cacheDir({'foo': '1.0.0'}).validate();
+ await d.cacheDir({'foo': '2.0.0'}).validate();
+
+ await hostedHashesCache([
+ file('foo-1.0.0.sha256', await server.peekArchiveSha256('foo', '1.0.0')),
+ ]).validate();
+
+ await hostedHashesCache([
+ file('foo-2.0.0.sha256', await server.peekArchiveSha256('foo', '2.0.0')),
+ ]).validate();
+
+ await pubGet(args: ['--offline']);
+ });
+
+ test(
+ 'installs package according to PUB_HOSTED_URL even on non-offical server',
+ () async {
+ final server = await servePackages();
+ server.serve('foo', '1.0.0');
+
+ final archivePath = p.join(sandbox, 'archive');
+
+ File(archivePath).writeAsBytesSync(await readBytes(
+ Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz')));
+ await runPub(
+ args: ['cache', 'preload', archivePath],
+ // By having pub.dev be the "official" server the test-server (localhost)
+ // is considered non-official. Test that the output mentions that we
+ // are installing to a non-official server.
+ environment: {'_PUB_TEST_DEFAULT_HOSTED_URL': 'pub.dev'},
+ output: allOf([
+ contains(
+ 'Installed $archivePath in cache as foo 1.0.0 from ${server.url}.')
+ ]),
+ );
+ await d.cacheDir({'foo': '1.0.0'}).validate();
+ });
+
+ test('overwrites existing entry in cache', () async {
+ final server = await servePackages();
+ server.serve('foo', '1.0.0', contents: [file('old-file.txt')]);
+
+ final archivePath = p.join(sandbox, 'archive');
+
+ File(archivePath).writeAsBytesSync(
+ await readBytes(
+ Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz'),
+ ),
+ );
+ await runPub(
+ args: ['cache', 'preload', archivePath],
+ environment: {'_PUB_TEST_DEFAULT_HOSTED_URL': server.url},
+ output:
+ allOf([contains('Installed $archivePath in cache as foo 1.0.0.')]),
+ );
+
+ server.serve('foo', '1.0.0', contents: [file('new-file.txt')]);
+
+ File(archivePath).writeAsBytesSync(
+ await readBytes(
+ Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz'),
+ ),
+ );
+
+ File(archivePath).writeAsBytesSync(
+ await readBytes(
+ Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz'),
+ ),
+ );
+
+ await runPub(
+ args: ['cache', 'preload', archivePath],
+ environment: {'_PUB_TEST_DEFAULT_HOSTED_URL': server.url},
+ output:
+ allOf([contains('Installed $archivePath in cache as foo 1.0.0.')]),
+ );
+ await hostedCache([
+ dir('foo-1.0.0', [file('new-file.txt'), nothing('old-file.txt')])
+ ]).validate();
+ });
+
+ test('handles missing archive', () async {
+ final archivePath = p.join(sandbox, 'archive');
+ await runPub(
+ args: ['cache', 'preload', archivePath],
+ error: contains('Could not find file $archivePath.'),
+ exitCode: 1,
+ );
+ });
+
+ test('handles broken archives', () async {
+ final archivePath = p.join(sandbox, 'archive');
+ File(archivePath).writeAsBytesSync('garbage'.codeUnits);
+ await runPub(
+ args: ['cache', 'preload', archivePath],
+ error:
+ contains('Failed to extract `$archivePath`: Filter error, bad data.'),
+ exitCode: DATA,
+ );
+ });
+
+ test('handles missing pubspec.yaml in archive', () async {
+ final archivePath = p.join(sandbox, 'archive');
+
+ // Create a tar.gz with a single file (and no pubspec.yaml).
+ File(archivePath).writeAsBytesSync(
+ await tarFromDescriptors([d.file('foo.txt')]).expand((x) => x).toList(),
+ );
+
+ await runPub(
+ args: ['cache', 'preload', archivePath],
+ error: contains(
+ 'Found no `pubspec.yaml` in $archivePath. Is it a valid pub package archive?',
+ ),
+ exitCode: 1,
+ );
+ });
+
+ test('handles broken pubspec.yaml in archive', () async {
+ final archivePath = p.join(sandbox, 'archive');
+
+ File(archivePath).writeAsBytesSync(
+ await tarFromDescriptors([d.file('pubspec.yaml', '{}')])
+ .expand((x) => x)
+ .toList());
+
+ await runPub(
+ args: ['cache', 'preload', archivePath],
+ error: contains(
+ 'Failed to load `pubspec.yaml` from `$archivePath`: Error on line 1, column 1',
+ ),
+ exitCode: 1,
+ );
+ });
+}
diff --git a/test/package_server.dart b/test/package_server.dart
index 06a8d1b..d69d0ef 100644
--- a/test/package_server.dart
+++ b/test/package_server.dart
@@ -11,7 +11,6 @@
import 'package:path/path.dart' as p;
import 'package:pub/src/crc32c.dart';
import 'package:pub/src/source/hosted.dart';
-import 'package:pub/src/third_party/tar/tar.dart';
import 'package:pub/src/utils.dart' show hexEncode;
import 'package:pub_semver/pub_semver.dart';
import 'package:shelf/shelf.dart' as shelf;
@@ -239,60 +238,10 @@
package.versions[version] = _ServedPackageVersion(
pubspecFields,
headers: headers,
- contents: () {
- final entries = <TarEntry>[];
-
- void addDescriptor(d.Descriptor descriptor, String path) {
- if (descriptor is d.DirectoryDescriptor) {
- for (final e in descriptor.contents) {
- addDescriptor(e, p.posix.join(path, descriptor.name));
- }
- } else {
- entries.add(
- TarEntry(
- TarHeader(
- // Ensure paths in tar files use forward slashes
- name: p.posix.join(path, descriptor.name),
- // We want to keep executable bits, but otherwise use the default
- // file mode
- mode: 420,
- // size: 100,
- modified: DateTime.fromMicrosecondsSinceEpoch(0),
- userName: 'pub',
- groupName: 'pub',
- ),
- (descriptor as d.FileDescriptor).readAsBytes(),
- ),
- );
- }
- }
-
- for (final e in contents ?? <d.Descriptor>[]) {
- addDescriptor(e, '');
- }
- return _replaceOs(Stream.fromIterable(entries)
- .transform(tarWriterWith(format: OutputFormat.gnuLongName))
- .transform(gzip.encoder));
- },
+ contents: () => tarFromDescriptors(contents ?? []),
);
}
- /// Replaces the entry at index 9 in [stream] with a 0. This replaces the os
- /// entry of a gzip stream, giving us the same stream and thius stable testing
- /// on all platforms.
- ///
- /// See https://www.rfc-editor.org/rfc/rfc1952 section 2.3 for information
- /// about the OS header.
- Stream<List<int>> _replaceOs(Stream<List<int>> stream) async* {
- final bytesBuilder = BytesBuilder();
- await for (final t in stream) {
- bytesBuilder.add(t);
- }
- final result = bytesBuilder.toBytes();
- result[9] = 0;
- yield result;
- }
-
// Mark a package discontinued.
void discontinue(String name,
{bool isDiscontinued = true, String? replacementText}) {
diff --git a/test/test_pub.dart b/test/test_pub.dart
index 58da455..6a2e67b 100644
--- a/test/test_pub.dart
+++ b/test/test_pub.dart
@@ -9,9 +9,10 @@
/// library provides an API to build tests like that.
import 'dart:convert';
import 'dart:core';
-import 'dart:io';
+import 'dart:io' hide BytesBuilder;
import 'dart:isolate';
import 'dart:math';
+import 'dart:typed_data';
import 'package:async/async.dart';
import 'package:http/testing.dart';
@@ -26,6 +27,7 @@
import 'package:pub/src/package_name.dart';
import 'package:pub/src/source/hosted.dart';
import 'package:pub/src/system_cache.dart';
+import 'package:pub/src/third_party/tar/tar.dart';
import 'package:pub/src/utils.dart';
import 'package:pub/src/validator.dart';
import 'package:pub_semver/pub_semver.dart';
@@ -974,3 +976,54 @@
'PATH': '$binFolder$separator${Platform.environment['PATH']}',
};
}
+
+Stream<List<int>> tarFromDescriptors(Iterable<d.Descriptor> contents) {
+ final entries = <TarEntry>[];
+ void addDescriptor(d.Descriptor descriptor, String path) {
+ if (descriptor is d.DirectoryDescriptor) {
+ for (final e in descriptor.contents) {
+ addDescriptor(e, p.posix.join(path, descriptor.name));
+ }
+ } else {
+ entries.add(
+ TarEntry(
+ TarHeader(
+ // Ensure paths in tar files use forward slashes
+ name: p.posix.join(path, descriptor.name),
+ // We want to keep executable bits, but otherwise use the default
+ // file mode
+ mode: 420,
+ // size: 100,
+ modified: DateTime.fromMicrosecondsSinceEpoch(0),
+ userName: 'pub',
+ groupName: 'pub',
+ ),
+ (descriptor as d.FileDescriptor).readAsBytes(),
+ ),
+ );
+ }
+ }
+
+ for (final e in contents) {
+ addDescriptor(e, '');
+ }
+ return _replaceOs(Stream.fromIterable(entries)
+ .transform(tarWriterWith(format: OutputFormat.gnuLongName))
+ .transform(gzip.encoder));
+}
+
+/// Replaces the entry at index 9 in [stream] with a 0. This replaces the os
+/// entry of a gzip stream, giving us the same stream and thius stable testing
+/// on all platforms.
+///
+/// See https://www.rfc-editor.org/rfc/rfc1952 section 2.3 for information
+/// about the OS header.
+Stream<List<int>> _replaceOs(Stream<List<int>> stream) async* {
+ final bytesBuilder = BytesBuilder();
+ await for (final t in stream) {
+ bytesBuilder.add(t);
+ }
+ final result = bytesBuilder.toBytes();
+ result[9] = 0;
+ yield result;
+}
