New command `dart pub cache preload` (#3636)
diff --git a/lib/src/command/cache.dart b/lib/src/command/cache.dart index 12c4fbf..43e8c5a 100644 --- a/lib/src/command/cache.dart +++ b/lib/src/command/cache.dart
@@ -6,6 +6,7 @@ import 'cache_add.dart'; import 'cache_clean.dart'; import 'cache_list.dart'; +import 'cache_preload.dart'; import 'cache_repair.dart'; /// Handles the `cache` pub command. @@ -22,5 +23,8 @@ addSubcommand(CacheListCommand()); addSubcommand(CacheCleanCommand()); addSubcommand(CacheRepairCommand()); + addSubcommand( + CachePreloadCommand(), + ); } }
diff --git a/lib/src/command/cache_preload.dart b/lib/src/command/cache_preload.dart new file mode 100644 index 0000000..54b7867 --- /dev/null +++ b/lib/src/command/cache_preload.dart
@@ -0,0 +1,49 @@ +// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:async'; + +import '../command.dart'; +import '../io.dart'; +import '../log.dart' as log; +import '../source/hosted.dart'; +import '../utils.dart'; + +/// Handles the `cache preload` pub command. +class CachePreloadCommand extends PubCommand { + @override + String get name => 'preload'; + @override + String get description => 'Install packages from a .tar.gz archive.'; + @override + String get argumentsDescription => '<package1.tar.gz> ...'; + @override + String get docUrl => 'https://dart.dev/tools/pub/cmd/pub-cache'; + + /// The `cache preload` command is hidden by default, because it's really only intended for + /// `flutter` to use when pre-loading `PUB_CACHE` after being installed from `zip` archive. + @override + bool get hidden => true; + + @override + Future<void> runProtected() async { + // Make sure there is a package. + if (argResults.rest.isEmpty) { + usageException('No package to preload given.'); + } + + for (String packagePath in argResults.rest) { + if (!fileExists(packagePath)) { + fail('Could not find file $packagePath.'); + } + } + for (String archivePath in argResults.rest) { + final id = await cache.hosted.preloadPackage(archivePath, cache); + final url = (id.description.description as HostedDescription).url; + + final fromPart = HostedSource.isFromPubDev(id) ? '' : ' from $url'; + log.message('Installed $archivePath in cache as $id$fromPart.'); + } + } +}
diff --git a/lib/src/source/hosted.dart b/lib/src/source/hosted.dart index 4e8a939..5922e3c 100644 --- a/lib/src/source/hosted.dart +++ b/lib/src/source/hosted.dart
@@ -129,6 +129,11 @@ static bool isPubDevUrl(String url) { final origin = Uri.parse(url).origin; + // Allow the defaultHostedUrl to be overriden when running from tests + if (runningFromTest && + io.Platform.environment['_PUB_TEST_DEFAULT_HOSTED_URL'] != null) { + return origin == io.Platform.environment['_PUB_TEST_DEFAULT_HOSTED_URL']; + } return origin == pubDevUrl || origin == pubDartlangUrl; } @@ -994,7 +999,7 @@ versions.firstWhereOrNull((i) => i.version == id.version); final packageName = id.name; final version = id.version; - late Uint8List contentHash; + late final Uint8List contentHash; if (versionInfo == null) { throw PackageNotFoundException( 'Package $packageName has no version $version'); @@ -1032,13 +1037,8 @@ See $contentHashesDocumentationUrl. '''); } - final path = hashPath(id, cache); - ensureDir(p.dirname(path)); - writeTextFile( - path, - hexEncode(actualHash.bytes), - ); contentHash = Uint8List.fromList(actualHash.bytes); + writeHash(id, cache, contentHash); } // It is important that we do not compare against id.description.sha256, @@ -1086,9 +1086,14 @@ ); var tempDir = cache.createTempDir(); - await extractTarGz(readBinaryFileAsStream(archivePath), tempDir); + try { + await extractTarGz(readBinaryFileAsStream(archivePath), tempDir); - ensureDir(p.dirname(destPath)); + ensureDir(p.dirname(destPath)); + } catch (e) { + deleteEntry(tempDir); + rethrow; + } // Now that the get has succeeded, move it to the real location in the // cache. // @@ -1100,6 +1105,84 @@ }); } + /// Writes the contenthash for [id] in the cache. + void writeHash(PackageId id, SystemCache cache, List<int> bytes) { + final path = hashPath(id, cache); + ensureDir(p.dirname(path)); + writeTextFile( + path, + hexEncode(bytes), + ); + } + + /// Installs a tar.gz file in [archivePath] as if it was downloaded from a + /// package repository. + /// + /// The name, version and repository are decided from the pubspec.yaml that + /// must be present in the archive. + Future<PackageId> preloadPackage( + String archivePath, SystemCache cache) async { + // Extract to a temp-folder and do atomic rename to preserve the integrity + // of the cache. + late final Uint8List contentHash; + + var tempDir = cache.createTempDir(); + final PackageId id; + try { + try { + // We read the file twice, once to compute the hash, and once to extract + // the archive. + // + // It would be desirable to read the file only once, but the tar + // extraction closes the stream early making things tricky to get right. + contentHash = Uint8List.fromList( + (await sha256.bind(readBinaryFileAsStream(archivePath)).first) + .bytes); + await extractTarGz(readBinaryFileAsStream(archivePath), tempDir); + } on FormatException catch (e) { + dataError('Failed to extract `$archivePath`: ${e.message}.'); + } + if (!fileExists(p.join(tempDir, 'pubspec.yaml'))) { + fail( + 'Found no `pubspec.yaml` in $archivePath. Is it a valid pub package archive?'); + } + final Pubspec pubspec; + try { + pubspec = Pubspec.load(tempDir, cache.sources); + final errors = pubspec.allErrors; + if (errors.isNotEmpty) { + throw errors.first; + } + } on Exception catch (e) { + fail('Failed to load `pubspec.yaml` from `$archivePath`: $e.'); + } + // Reconstruct the PackageId from the extracted pubspec.yaml. + id = PackageId( + pubspec.name, + pubspec.version, + ResolvedHostedDescription( + HostedDescription( + pubspec.name, + validateAndNormalizeHostedUrl(cache.hosted.defaultUrl).toString(), + ), + sha256: contentHash, + ), + ); + } catch (e) { + deleteEntry(tempDir); + rethrow; + } + final packageDir = getDirectoryInCache(id, cache); + if (dirExists(packageDir)) { + log.fine( + 'Cache entry for ${id.name}-${id.version} already exists. Replacing.'); + deleteEntry(packageDir); + } + tryRenameDir(tempDir, packageDir); + writeHash(id, cache, contentHash); + return id; + } + /// When an error occurs trying to read something about [package] from [hostedUrl], /// this tries to translate into a more user friendly error message. ///
diff --git a/test/cache/preload_test.dart b/test/cache/preload_test.dart new file mode 100644 index 0000000..3bf5f64 --- /dev/null +++ b/test/cache/preload_test.dart
@@ -0,0 +1,180 @@ +// Copyright (c) 2013, the Dart project authors. Please see the AUTHORS file +// for details. All rights reserved. Use of this source code is governed by a +// BSD-style license that can be found in the LICENSE file. + +import 'dart:io'; + +import 'package:http/http.dart'; +import 'package:path/path.dart' as p; +import 'package:pub/src/exit_codes.dart'; +import 'package:test/test.dart'; + +import '../descriptor.dart' as d; +import '../descriptor.dart'; +import '../test_pub.dart'; + +void main() { + test('adds correct entries to cache and stores the content-hash', () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + server.serve('foo', '2.0.0'); + + await appDir({'foo': '^2.0.0'}).create(); + // Do a `pub get` here to create a lock file in order to validate we later can + // `pub get --offline` with packages installed by `preload`. + await pubGet(); + + await runPub(args: ['cache', 'clean', '-f']); + + final archivePath1 = p.join(sandbox, 'foo-1.0.0-archive.tar.gz'); + final archivePath2 = p.join(sandbox, 'foo-2.0.0-archive.tar.gz'); + + File(archivePath1).writeAsBytesSync(await readBytes( + Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz'))); + File(archivePath2).writeAsBytesSync(await readBytes( + Uri.parse(server.url).resolve('packages/foo/versions/2.0.0.tar.gz'))); + await runPub( + args: ['cache', 'preload', archivePath1, archivePath2], + environment: {'_PUB_TEST_DEFAULT_HOSTED_URL': server.url}, + output: allOf( + [ + contains('Installed $archivePath1 in cache as foo 1.0.0.'), + contains('Installed $archivePath2 in cache as foo 2.0.0.'), + ], + ), + ); + await d.cacheDir({'foo': '1.0.0'}).validate(); + await d.cacheDir({'foo': '2.0.0'}).validate(); + + await hostedHashesCache([ + file('foo-1.0.0.sha256', await server.peekArchiveSha256('foo', '1.0.0')), + ]).validate(); + + await hostedHashesCache([ + file('foo-2.0.0.sha256', await server.peekArchiveSha256('foo', '2.0.0')), + ]).validate(); + + await pubGet(args: ['--offline']); + }); + + test( + 'installs package according to PUB_HOSTED_URL even on non-offical server', + () async { + final server = await servePackages(); + server.serve('foo', '1.0.0'); + + final archivePath = p.join(sandbox, 'archive'); + + File(archivePath).writeAsBytesSync(await readBytes( + Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz'))); + await runPub( + args: ['cache', 'preload', archivePath], + // By having pub.dev be the "official" server the test-server (localhost) + // is considered non-official. Test that the output mentions that we + // are installing to a non-official server. + environment: {'_PUB_TEST_DEFAULT_HOSTED_URL': 'pub.dev'}, + output: allOf([ + contains( + 'Installed $archivePath in cache as foo 1.0.0 from ${server.url}.') + ]), + ); + await d.cacheDir({'foo': '1.0.0'}).validate(); + }); + + test('overwrites existing entry in cache', () async { + final server = await servePackages(); + server.serve('foo', '1.0.0', contents: [file('old-file.txt')]); + + final archivePath = p.join(sandbox, 'archive'); + + File(archivePath).writeAsBytesSync( + await readBytes( + Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz'), + ), + ); + await runPub( + args: ['cache', 'preload', archivePath], + environment: {'_PUB_TEST_DEFAULT_HOSTED_URL': server.url}, + output: + allOf([contains('Installed $archivePath in cache as foo 1.0.0.')]), + ); + + server.serve('foo', '1.0.0', contents: [file('new-file.txt')]); + + File(archivePath).writeAsBytesSync( + await readBytes( + Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz'), + ), + ); + + File(archivePath).writeAsBytesSync( + await readBytes( + Uri.parse(server.url).resolve('packages/foo/versions/1.0.0.tar.gz'), + ), + ); + + await runPub( + args: ['cache', 'preload', archivePath], + environment: {'_PUB_TEST_DEFAULT_HOSTED_URL': server.url}, + output: + allOf([contains('Installed $archivePath in cache as foo 1.0.0.')]), + ); + await hostedCache([ + dir('foo-1.0.0', [file('new-file.txt'), nothing('old-file.txt')]) + ]).validate(); + }); + + test('handles missing archive', () async { + final archivePath = p.join(sandbox, 'archive'); + await runPub( + args: ['cache', 'preload', archivePath], + error: contains('Could not find file $archivePath.'), + exitCode: 1, + ); + }); + + test('handles broken archives', () async { + final archivePath = p.join(sandbox, 'archive'); + File(archivePath).writeAsBytesSync('garbage'.codeUnits); + await runPub( + args: ['cache', 'preload', archivePath], + error: + contains('Failed to extract `$archivePath`: Filter error, bad data.'), + exitCode: DATA, + ); + }); + + test('handles missing pubspec.yaml in archive', () async { + final archivePath = p.join(sandbox, 'archive'); + + // Create a tar.gz with a single file (and no pubspec.yaml). + File(archivePath).writeAsBytesSync( + await tarFromDescriptors([d.file('foo.txt')]).expand((x) => x).toList(), + ); + + await runPub( + args: ['cache', 'preload', archivePath], + error: contains( + 'Found no `pubspec.yaml` in $archivePath. Is it a valid pub package archive?', + ), + exitCode: 1, + ); + }); + + test('handles broken pubspec.yaml in archive', () async { + final archivePath = p.join(sandbox, 'archive'); + + File(archivePath).writeAsBytesSync( + await tarFromDescriptors([d.file('pubspec.yaml', '{}')]) + .expand((x) => x) + .toList()); + + await runPub( + args: ['cache', 'preload', archivePath], + error: contains( + 'Failed to load `pubspec.yaml` from `$archivePath`: Error on line 1, column 1', + ), + exitCode: 1, + ); + }); +}
diff --git a/test/package_server.dart b/test/package_server.dart index 06a8d1b..d69d0ef 100644 --- a/test/package_server.dart +++ b/test/package_server.dart
@@ -11,7 +11,6 @@ import 'package:path/path.dart' as p; import 'package:pub/src/crc32c.dart'; import 'package:pub/src/source/hosted.dart'; -import 'package:pub/src/third_party/tar/tar.dart'; import 'package:pub/src/utils.dart' show hexEncode; import 'package:pub_semver/pub_semver.dart'; import 'package:shelf/shelf.dart' as shelf; @@ -239,60 +238,10 @@ package.versions[version] = _ServedPackageVersion( pubspecFields, headers: headers, - contents: () { - final entries = <TarEntry>[]; - - void addDescriptor(d.Descriptor descriptor, String path) { - if (descriptor is d.DirectoryDescriptor) { - for (final e in descriptor.contents) { - addDescriptor(e, p.posix.join(path, descriptor.name)); - } - } else { - entries.add( - TarEntry( - TarHeader( - // Ensure paths in tar files use forward slashes - name: p.posix.join(path, descriptor.name), - // We want to keep executable bits, but otherwise use the default - // file mode - mode: 420, - // size: 100, - modified: DateTime.fromMicrosecondsSinceEpoch(0), - userName: 'pub', - groupName: 'pub', - ), - (descriptor as d.FileDescriptor).readAsBytes(), - ), - ); - } - } - - for (final e in contents ?? <d.Descriptor>[]) { - addDescriptor(e, ''); - } - return _replaceOs(Stream.fromIterable(entries) - .transform(tarWriterWith(format: OutputFormat.gnuLongName)) - .transform(gzip.encoder)); - }, + contents: () => tarFromDescriptors(contents ?? []), ); } - /// Replaces the entry at index 9 in [stream] with a 0. This replaces the os - /// entry of a gzip stream, giving us the same stream and thius stable testing - /// on all platforms. - /// - /// See https://www.rfc-editor.org/rfc/rfc1952 section 2.3 for information - /// about the OS header. - Stream<List<int>> _replaceOs(Stream<List<int>> stream) async* { - final bytesBuilder = BytesBuilder(); - await for (final t in stream) { - bytesBuilder.add(t); - } - final result = bytesBuilder.toBytes(); - result[9] = 0; - yield result; - } - // Mark a package discontinued. void discontinue(String name, {bool isDiscontinued = true, String? replacementText}) {
diff --git a/test/test_pub.dart b/test/test_pub.dart index 58da455..6a2e67b 100644 --- a/test/test_pub.dart +++ b/test/test_pub.dart
@@ -9,9 +9,10 @@ /// library provides an API to build tests like that. import 'dart:convert'; import 'dart:core'; -import 'dart:io'; +import 'dart:io' hide BytesBuilder; import 'dart:isolate'; import 'dart:math'; +import 'dart:typed_data'; import 'package:async/async.dart'; import 'package:http/testing.dart'; @@ -26,6 +27,7 @@ import 'package:pub/src/package_name.dart'; import 'package:pub/src/source/hosted.dart'; import 'package:pub/src/system_cache.dart'; +import 'package:pub/src/third_party/tar/tar.dart'; import 'package:pub/src/utils.dart'; import 'package:pub/src/validator.dart'; import 'package:pub_semver/pub_semver.dart'; @@ -974,3 +976,54 @@ 'PATH': '$binFolder$separator${Platform.environment['PATH']}', }; } + +Stream<List<int>> tarFromDescriptors(Iterable<d.Descriptor> contents) { + final entries = <TarEntry>[]; + void addDescriptor(d.Descriptor descriptor, String path) { + if (descriptor is d.DirectoryDescriptor) { + for (final e in descriptor.contents) { + addDescriptor(e, p.posix.join(path, descriptor.name)); + } + } else { + entries.add( + TarEntry( + TarHeader( + // Ensure paths in tar files use forward slashes + name: p.posix.join(path, descriptor.name), + // We want to keep executable bits, but otherwise use the default + // file mode + mode: 420, + // size: 100, + modified: DateTime.fromMicrosecondsSinceEpoch(0), + userName: 'pub', + groupName: 'pub', + ), + (descriptor as d.FileDescriptor).readAsBytes(), + ), + ); + } + } + + for (final e in contents) { + addDescriptor(e, ''); + } + return _replaceOs(Stream.fromIterable(entries) + .transform(tarWriterWith(format: OutputFormat.gnuLongName)) + .transform(gzip.encoder)); +} + +/// Replaces the entry at index 9 in [stream] with a 0. This replaces the os +/// entry of a gzip stream, giving us the same stream and thius stable testing +/// on all platforms. +/// +/// See https://www.rfc-editor.org/rfc/rfc1952 section 2.3 for information +/// about the OS header. +Stream<List<int>> _replaceOs(Stream<List<int>> stream) async* { + final bytesBuilder = BytesBuilder(); + await for (final t in stream) { + bytesBuilder.add(t); + } + final result = bytesBuilder.toBytes(); + result[9] = 0; + yield result; +}