Allow publishing to and from a .tar.gz archive (#4119)
diff --git a/lib/src/command/lish.dart b/lib/src/command/lish.dart
index a9aa9d2..f6ca1a4 100644
--- a/lib/src/command/lish.dart
+++ b/lib/src/command/lish.dart
@@ -3,7 +3,9 @@
// BSD-style license that can be found in the LICENSE file.
import 'dart:async';
+import 'dart:convert';
import 'dart:io';
+import 'dart:typed_data';
import 'package:http/http.dart' as http;
@@ -12,11 +14,12 @@
import '../command.dart';
import '../command_runner.dart';
import '../exceptions.dart' show DataException;
-import '../exit_codes.dart' as exit_codes;
+import '../exit_codes.dart';
import '../http.dart';
import '../io.dart';
import '../log.dart' as log;
import '../oauth2.dart' as oauth2;
+import '../pubspec.dart';
import '../solver/type.dart';
import '../source/hosted.dart' show validateAndNormalizeHostedUrl;
import '../utils.dart';
@@ -36,7 +39,7 @@
bool get takesArguments => false;
/// The URL of the server to which to upload the package.
- late final Uri host = () {
+ Uri computeHost(Pubspec pubspec) {
// An explicit argument takes precedence.
if (argResults.wasParsed('server')) {
try {
@@ -47,8 +50,8 @@
}
// Otherwise, use the one specified in the pubspec.
- final publishTo = entrypoint.root.pubspec.publishTo;
- if (publishTo != null) {
+ final publishTo = pubspec.publishTo;
+ if (publishTo != null && publishTo != 'none') {
try {
return validateAndNormalizeHostedUrl(publishTo);
} on FormatException catch (e) {
@@ -58,7 +61,7 @@
// Use the default server if nothing else is specified
return Uri.parse(cache.hosted.defaultUrl);
- }();
+ }
/// Whether the publish is just a preview.
bool get dryRun => argResults.flag('dry-run');
@@ -68,6 +71,10 @@
bool get skipValidation => argResults.flag('skip-validation');
+ late final String? _fromArchive =
+ argResults.optionWithoutDefault('from-archive');
+ late final String? _toArchive = argResults.optionWithoutDefault('to-archive');
+
LishCommand() {
argParser.addFlag(
'dry-run',
@@ -92,6 +99,19 @@
help: 'The package server to which to upload this package.',
hide: true,
);
+ argParser.addOption(
+ 'to-archive',
+ help: 'Create a .tar.gz archive instead of publishing to server',
+ valueHelp: '[archive.tar.gz]',
+ hide: true,
+ );
+ argParser.addOption(
+ 'from-archive',
+ help:
+ 'Publish from a .tar.gz archive instead of current folder. Implies `--skip-validation`.',
+ valueHelp: '[archive.tar.gz]',
+ hide: true,
+ );
argParser.addOption(
'directory',
@@ -104,6 +124,7 @@
Future<void> _publishUsingClient(
List<int> packageBytes,
http.Client client,
+ Uri host,
) async {
Uri? cloudStorageUrl;
@@ -189,7 +210,7 @@
}
}
- Future<void> _publish(List<int> packageBytes) async {
+ Future<void> _publish(List<int> packageBytes, Uri host) async {
try {
final officialPubServers = {
'https://pub.dev',
@@ -217,12 +238,12 @@
// This allows us to use `dart pub token add` to inject a token for use
// with the official servers.
await oauth2.withClient((client) {
- return _publishUsingClient(packageBytes, client);
+ return _publishUsingClient(packageBytes, client, host);
});
} else {
// For third party servers using bearer authentication client
await withAuthenticatedClient(cache, host, (client) {
- return _publishUsingClient(packageBytes, client);
+ return _publishUsingClient(packageBytes, client, host);
});
}
} on PubHttpResponseException catch (error) {
@@ -235,8 +256,7 @@
}
}
- @override
- Future runProtected() async {
+ Future<void> _validateArgs() async {
if (argResults.wasParsed('server')) {
await log.errorsOnlyUnlessTerminal(() {
log.message(
@@ -251,18 +271,26 @@
usageException('Cannot use both --force and --dry-run.');
}
- if (entrypoint.root.pubspec.isPrivate) {
- dataError('A private package cannot be published.\n'
- 'You can enable this by changing the "publish_to" field in your '
- 'pubspec.');
+ if (_fromArchive != null && _toArchive != null) {
+ usageException('Cannot use both --from-archive and --to-archive.');
}
- if (!skipValidation) {
- await entrypoint.acquireDependencies(SolveType.get);
- } else {
+ if (_fromArchive != null && dryRun) {
+ usageException('Cannot use both --from-archive and --dry-run.');
+ }
+
+ if (_toArchive != null && force) {
+ usageException('Cannot use both --to-archive and --force.');
+ }
+ }
+
+ Future<_Publication> _publicationFromEntrypoint() async {
+ if (skipValidation) {
log.warning(
'Running with `skip-validation`. No client-side validation is done.',
);
+ } else {
+ await entrypoint.acquireDependencies(SolveType.get);
}
var files = entrypoint.root.listFiles();
@@ -270,45 +298,79 @@
// Show the package contents so the user can verify they look OK.
var package = entrypoint.root;
+ final host = computeHost(package.pubspec);
log.message(
'Publishing ${package.name} ${package.version} to $host:\n'
'${tree.fromFiles(files, baseDir: entrypoint.rootDir, showFileSizes: true)}',
);
- var packageBytes =
+ final packageBytes =
await createTarGz(files, baseDir: entrypoint.rootDir).toBytes();
+
log.message(
'\nTotal compressed archive size: ${_readableFileSize(packageBytes.length)}.\n',
);
- // Validate the package.
- var isValid = skipValidation
- ? true
- : await _validate(
- packageBytes.length,
- files,
- );
- if (!isValid) {
- overrideExitCode(exit_codes.DATA);
- return;
- } else if (dryRun) {
+ final validationResult =
+ skipValidation ? null : await _validate(packageBytes, files, host);
+
+ if (dryRun) {
log.message('The server may enforce additional checks.');
- return;
- } else {
- await _publish(packageBytes);
}
+ return _Publication(
+ packageBytes: packageBytes,
+ warningCount: validationResult?.warningsCount ?? 0,
+ hintCount: validationResult?.hintsCount ?? 0,
+ pubspec: package.pubspec,
+ );
}
- /// Returns the value associated with [key] in [map]. Throws a user-friendly
- /// error if [map] doesn't contain [key].
- dynamic _expectField(Map map, String key, http.Response response) {
- if (map.containsKey(key)) return map[key];
- invalidServerResponse(response);
+ Future<_Publication> _publicationFromArchive(String archive) async {
+ final Uint8List packageBytes;
+ try {
+ log.message('Publishing from archive: $_fromArchive');
+
+ packageBytes = readBinaryFile(archive);
+ } on FileSystemException catch (e) {
+ dataError(
+ 'Failed reading archive file: $e)',
+ );
+ }
+ final Pubspec pubspec;
+ try {
+ pubspec = Pubspec.parse(
+ utf8.decode(
+ await extractFileFromTarGz(
+ Stream.fromIterable([packageBytes]),
+ 'pubspec.yaml',
+ ),
+ ),
+ cache.sources,
+ );
+ } on FormatException catch (e) {
+ dataError('Failed to read pubspec.yaml from archive: ${e.message}');
+ }
+ final host = computeHost(pubspec);
+ log.message('Publishing ${pubspec.name} ${pubspec.version} to $host.');
+ return _Publication(
+ packageBytes: packageBytes,
+ warningCount: 0,
+ hintCount: 0,
+ pubspec: pubspec,
+ );
}
- /// Validates the package. Completes to false if the upload should not
+ /// Validates the package.
+ ///
+ /// Throws if there are errors and the upload should not
/// proceed.
- Future<bool> _validate(int packageSize, List<String> files) async {
+ ///
+ /// Returns a summary of warnings and hints if there are any, otherwise `null`.
+ Future<({int warningsCount, int hintsCount})> _validate(
+ Uint8List packageBytes,
+ List<String> files,
+ Uri host,
+ ) async {
final hints = <String>[];
final warnings = <String>[];
final errors = <String>[];
@@ -317,7 +379,7 @@
'Validating package',
() async => await Validator.runAll(
entrypoint,
- packageSize,
+ packageBytes.length,
host,
files,
hints: hints,
@@ -327,46 +389,80 @@
);
if (errors.isNotEmpty) {
- log.error('Sorry, your package is missing '
+ dataError('Sorry, your package is missing '
"${(errors.length > 1) ? 'some requirements' : 'a requirement'} "
"and can't be published yet.\nFor more information, see: "
'https://dart.dev/tools/pub/cmd/pub-lish.\n');
- return false;
}
- if (force) return true;
+ return (warningsCount: warnings.length, hintsCount: hints.length);
+ }
- String formatWarningCount() {
- final hintText = hints.isEmpty
- ? ''
- : ' and ${hints.length} ${pluralize('hint', hints.length)}';
- return '\nPackage has ${warnings.length} '
- '${pluralize('warning', warnings.length)}$hintText.';
- }
-
- if (dryRun) {
- log.warning(formatWarningCount());
- return warnings.isEmpty;
- }
-
+ /// Asks the user for confirmation of uploading [package].
+ ///
+ /// Skips asking if [force].
+ /// Throws if user didn't confirm.
+ Future<void> _confirmUpload(_Publication package, Uri host) async {
+ if (force) return;
log.message('\nPublishing is forever; packages cannot be unpublished.'
- '\nPolicy details are available at https://pub.dev/policy');
+ '\nPolicy details are available at https://pub.dev/policy\n');
- final package = entrypoint.root;
var message =
- 'Do you want to publish ${package.name} ${package.version} to $host';
-
- if (warnings.isNotEmpty || hints.isNotEmpty) {
- final warning = formatWarningCount();
- message = '${log.bold(log.red(warning))}. $message';
+ 'Do you want to publish ${package.pubspec.name} ${package.pubspec.version} to $host';
+ if (package.hintCount != 0 || package.warningCount != 0) {
+ message = '${package.warningsCountMessage}. $message';
}
-
- var confirmed = await confirm('\n$message');
- if (!confirmed) {
- log.error('Package upload canceled.');
- return false;
+ if (!await confirm('\n$message')) {
+ dataError('Package upload canceled.');
}
- return true;
+ }
+
+ @override
+ Future runProtected() async {
+ await _validateArgs();
+ final publication = await (_fromArchive == null
+ ? _publicationFromEntrypoint()
+ : _publicationFromArchive(_fromArchive));
+ if (dryRun) {
+ log.warning(publication.warningsCountMessage);
+ if (publication.warningCount != 0) {
+ overrideExitCode(DATA);
+ }
+ return;
+ }
+ if (_toArchive == null) {
+ final host = computeHost(publication.pubspec);
+ if (publication.pubspec.isPrivate) {
+ dataError('A private package cannot be published.\n'
+ 'You can enable this by changing the "publish_to" field in your '
+ 'pubspec.');
+ }
+ await _confirmUpload(publication, host);
+
+ await _publish(publication.packageBytes, host);
+ } else {
+ if (dryRun) {
+ log.message('Would have written to $_toArchive.');
+ } else {
+ _writeUploadToArchive(publication, _toArchive);
+ }
+ }
+ }
+
+ void _writeUploadToArchive(_Publication publication, String archive) {
+ try {
+ writeBinaryFile(archive, publication.packageBytes);
+ } on FileSystemException catch (e) {
+ dataError('Failed writing archive: $e');
+ }
+ log.message('Wrote package archive at $_toArchive');
+ }
+
+ /// Returns the value associated with [key] in [map]. Throws a user-friendly
+ /// error if [map] doesn't contain [key].
+ dynamic _expectField(Map map, String key, http.Response response) {
+ if (map.containsKey(key)) return map[key];
+ invalidServerResponse(response);
}
}
@@ -381,3 +477,25 @@
return '<1 KB';
}
}
+
+class _Publication {
+ Uint8List packageBytes;
+ int warningCount;
+ int hintCount;
+
+ Pubspec pubspec;
+
+ String get warningsCountMessage {
+ final hintText =
+ hintCount == 0 ? '' : ' and $hintCount ${pluralize('hint', hintCount)}';
+ return '\nPackage has $warningCount '
+ '${pluralize('warning', warningCount)}$hintText.';
+ }
+
+ _Publication({
+ required this.packageBytes,
+ required this.warningCount,
+ required this.hintCount,
+ required this.pubspec,
+ });
+}
diff --git a/lib/src/io.dart b/lib/src/io.dart
index 3d105a1..79ebaba 100644
--- a/lib/src/io.dart
+++ b/lib/src/io.dart
@@ -9,7 +9,9 @@
import 'dart:collection';
import 'dart:convert';
import 'dart:io';
+import 'dart:typed_data';
+import 'package:async/async.dart';
import 'package:cli_util/cli_util.dart'
show EnvironmentNotFoundException, applicationConfigHome;
import 'package:http/http.dart' show ByteStream;
@@ -205,7 +207,7 @@
}
/// Reads the contents of the binary file [file].
-List<int> readBinaryFile(String file) {
+Uint8List readBinaryFile(String file) {
log.io('Reading binary file $file.');
var contents = File(file).readAsBytesSync();
log.io('Read ${contents.length} bytes from $file.');
@@ -239,6 +241,12 @@
File(file).writeAsStringSync(contents, encoding: encoding);
}
+/// Reads the contents of the binary file [file].
+void writeBinaryFile(String file, Uint8List data) {
+ log.io('Writing ${data.length} bytes to file $file.');
+ File(file).writeAsBytesSync(data);
+}
+
/// Creates [file] and writes [contents] to it.
///
/// If [dontLogContents] is `true`, the contents of the file will never be
@@ -666,9 +674,9 @@
/// should just be a fragment like, "Are you sure you want to proceed". The
/// default for an empty response, or any response not starting with `y` or `Y`
/// is false.
-Future<bool> confirm(String message) {
- log.fine('Showing confirm message: $message');
- return stdinPrompt('$message (y/N)?').then(RegExp(r'^[yY]').hasMatch);
+Future<bool> confirm(String message) async {
+ final reply = await stdinPrompt('$message (y/N)?');
+ return RegExp(r'^[yY]').hasMatch(reply);
}
/// Writes [prompt] and reads a line from stdin.
@@ -1000,8 +1008,32 @@
return server;
}
+/// Extracts a single file from a `.tar.gz` [stream].
+///
+/// [filename] should be the relative path inside the archive (with unix
+/// separators '/').
+///
+/// Throws a `FormatException` if that file did not exist.
+Future<Uint8List> extractFileFromTarGz(
+ Stream<List<int>> stream,
+ String filename,
+) async {
+ final reader = TarReader(stream.transform(gzip.decoder));
+ filename = path.posix.normalize(filename);
+ while (await reader.moveNext()) {
+ final entry = reader.current;
+ if (path.posix.normalize(entry.name) != filename) continue;
+ if (!(entry.type == TypeFlag.reg || entry.type == TypeFlag.regA)) {
+ // Can only read regular files.
+ throw FormatException('$filename is not a file');
+ }
+ return await collectBytes(entry.contents);
+ }
+ throw FormatException('Could not find $filename in archive');
+}
+
/// Extracts a `.tar.gz` file from [stream] to [destination].
-Future extractTarGz(Stream<List<int>> stream, String destination) async {
+Future<void> extractTarGz(Stream<List<int>> stream, String destination) async {
log.fine('Extracting .tar.gz stream to $destination.');
destination = path.absolute(destination);
diff --git a/test/descriptor.dart b/test/descriptor.dart
index 8fb6531..a91cc64 100644
--- a/test/descriptor.dart
+++ b/test/descriptor.dart
@@ -37,8 +37,12 @@
libPubspec('test_pkg', '1.0.0', sdk: '>=3.1.2 <=3.2.0', extras: extras);
/// Describes a package that passes all validation.
-DirectoryDescriptor validPackage({String version = '1.0.0'}) => dir(appPath, [
- validPubspec(extras: {'version': version}),
+DirectoryDescriptor validPackage({
+ String version = '1.0.0',
+ Map<String, Object?>? pubspecExtras,
+}) =>
+ dir(appPath, [
+ validPubspec(extras: {'version': version, ...?pubspecExtras}),
file('LICENSE', 'Eh, do what you want.'),
file('README.md', "This package isn't real."),
file('CHANGELOG.md', '# $version\nFirst version\n'),
diff --git a/test/lish/does_not_publish_if_private_test.dart b/test/lish/does_not_publish_if_private_test.dart
index c47610b..47f608d 100644
--- a/test/lish/does_not_publish_if_private_test.dart
+++ b/test/lish/does_not_publish_if_private_test.dart
@@ -10,13 +10,11 @@
void main() {
test('does not publish if the package is private', () async {
- var pkg = packageMap('test_pkg', '1.0.0');
- pkg['publish_to'] = 'none';
- await d.dir(appPath, [d.pubspec(pkg)]).create();
+ await d.validPackage(pubspecExtras: {'publish_to': 'none'}).create();
await runPub(
args: ['lish'],
- error: startsWith('A private package cannot be published.'),
+ error: contains('A private package cannot be published.'),
exitCode: exit_codes.DATA,
);
});
diff --git a/test/lish/does_not_publish_if_private_with_server_arg_test.dart b/test/lish/does_not_publish_if_private_with_server_arg_test.dart
index 689609b..c4fe7b9 100644
--- a/test/lish/does_not_publish_if_private_with_server_arg_test.dart
+++ b/test/lish/does_not_publish_if_private_with_server_arg_test.dart
@@ -12,13 +12,11 @@
test(
'does not publish if the package is private even if a server '
'argument is provided', () async {
- var pkg = packageMap('test_pkg', '1.0.0');
- pkg['publish_to'] = 'none';
- await d.dir(appPath, [d.pubspec(pkg)]).create();
+ await d.validPackage(pubspecExtras: {'publish_to': 'none'}).create();
await runPub(
args: ['lish'],
- error: startsWith('A private package cannot be published.'),
+ error: contains('A private package cannot be published.'),
environment: {'PUB_HOSTED_URL': 'http://example.com'},
exitCode: exit_codes.DATA,
);
diff --git a/test/lish/dry_run_errors_if_private_test.dart b/test/lish/dry_run_errors_if_private_test.dart
deleted file mode 100644
index c9b6c6a..0000000
--- a/test/lish/dry_run_errors_if_private_test.dart
+++ /dev/null
@@ -1,24 +0,0 @@
-// Copyright (c) 2014, the Dart project authors. Please see the AUTHORS file
-// for details. All rights reserved. Use of this source code is governed by a
-// BSD-style license that can be found in the LICENSE file.
-
-import 'package:pub/src/exit_codes.dart' as exit_codes;
-
-import 'package:test/test.dart';
-
-import '../descriptor.dart' as d;
-import '../test_pub.dart';
-
-void main() {
- test('dry-run shows an error if the package is private', () async {
- var pkg = packageMap('test_pkg', '1.0.0');
- pkg['publish_to'] = 'none';
- await d.dir(appPath, [d.pubspec(pkg)]).create();
-
- await runPub(
- args: ['lish', '--dry-run'],
- error: startsWith('A private package cannot be published.'),
- exitCode: exit_codes.DATA,
- );
- });
-}
diff --git a/test/lish/force_does_not_publish_if_private_test.dart b/test/lish/force_does_not_publish_if_private_test.dart
index edd3611..61dfb21 100644
--- a/test/lish/force_does_not_publish_if_private_test.dart
+++ b/test/lish/force_does_not_publish_if_private_test.dart
@@ -10,13 +10,11 @@
void main() {
test('force does not publish if the package is private', () async {
- var pkg = packageMap('test_pkg', '1.0.0');
- pkg['publish_to'] = 'none';
- await d.dir(appPath, [d.pubspec(pkg)]).create();
+ await d.validPackage(pubspecExtras: {'publish_to': 'none'}).create();
await runPub(
args: ['lish', '--force'],
- error: startsWith('A private package cannot be published.'),
+ error: contains('A private package cannot be published.'),
exitCode: exit_codes.DATA,
);
});
diff --git a/test/lish/publishing_to_and_from_archive_test.dart b/test/lish/publishing_to_and_from_archive_test.dart
new file mode 100644
index 0000000..8d3ee5c
--- /dev/null
+++ b/test/lish/publishing_to_and_from_archive_test.dart
@@ -0,0 +1,55 @@
+// Copyright (c) 2024, the Dart project authors. Please see the AUTHORS file
+// for details. All rights reserved. Use of this source code is governed by a
+// BSD-style license that can be found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io';
+
+import 'package:path/path.dart' as p;
+import 'package:pub/src/exit_codes.dart';
+import 'package:shelf/shelf.dart';
+import 'package:test/test.dart';
+
+import '../descriptor.dart' as d;
+import '../test_pub.dart';
+import 'utils.dart';
+
+void main() {
+ test('Can publish into and from archive', () async {
+ final server = await servePackages();
+ await d.validPackage().create();
+ await d.credentialsFile(server, 'access-token').create();
+ await runPub(
+ args: ['lish', '--to-archive', p.join('..', 'archive.tar.gz')],
+ output: contains(
+ 'Wrote package archive at ${p.join('..', 'archive.tar.gz')}',
+ ),
+ );
+ expect(File(d.path('archive.tar.gz')).existsSync(), isTrue);
+
+ server.expect('GET', '/create', (request) {
+ return Response.ok(
+ jsonEncode({
+ 'success': {'message': 'Package test_pkg 1.0.0 uploaded!'},
+ }),
+ );
+ });
+
+ final pub = await startPublish(
+ server,
+ args: ['--from-archive', 'archive.tar.gz'],
+ // Run outside the appPath to make sure we are not publishing that dir.
+ workingDirectory: d.sandbox,
+ );
+
+ expect(pub.stdout, emitsThrough('Publishing from archive: archive.tar.gz'));
+ await confirmPublish(pub);
+
+ handleUploadForm(server);
+ handleUpload(server);
+
+ expect(pub.stdout, emitsThrough(startsWith('Uploading...')));
+ expect(pub.stdout, emits('Package test_pkg 1.0.0 uploaded!'));
+ await pub.shouldExit(SUCCESS);
+ });
+}
diff --git a/test/lish/server_arg_does_not_override_private_test.dart b/test/lish/server_arg_does_not_override_private_test.dart
index f9d7bf8..e6cb779 100644
--- a/test/lish/server_arg_does_not_override_private_test.dart
+++ b/test/lish/server_arg_does_not_override_private_test.dart
@@ -11,13 +11,11 @@
void main() {
test('an explicit --server argument does not override privacy', () async {
- var pkg = packageMap('test_pkg', '1.0.0');
- pkg['publish_to'] = 'none';
- await d.dir(appPath, [d.pubspec(pkg)]).create();
+ await d.validPackage(pubspecExtras: {'publish_to': 'none'}).create();
await runPub(
args: ['lish', '--server', 'http://arg.com'],
- error: startsWith('A private package cannot be published.'),
+ error: contains('A private package cannot be published.'),
exitCode: exit_codes.DATA,
);
});
diff --git a/test/lish/skip_validation_test.dart b/test/lish/skip_validation_test.dart
index fa2e995..e9b3dc0 100644
--- a/test/lish/skip_validation_test.dart
+++ b/test/lish/skip_validation_test.dart
@@ -35,6 +35,8 @@
await servePackages();
var pub = await startPublish(globalServer, args: ['--skip-validation']);
+ await confirmPublish(pub);
+
handleUploadForm(globalServer);
handleUpload(globalServer);
diff --git a/test/test_pub.dart b/test/test_pub.dart
index cad7f30..7aa3cf6 100644
--- a/test/test_pub.dart
+++ b/test/test_pub.dart
@@ -402,6 +402,7 @@
bool overrideDefaultHostedServer = true,
Map<String, String>? environment,
String path = '',
+ String? workingDirectory,
}) async {
var tokenEndpoint = Uri.parse(server.url).resolve('/token').toString();
args = ['lish', ...?args];
@@ -415,6 +416,7 @@
'PUB_HOSTED_URL': server.url + path,
if (environment != null) ...environment,
},
+ workingDirectory: workingDirectory,
);
}
