| import 'dart:async'; |
| import 'dart:convert'; |
| import 'dart:typed_data'; |
| |
| import 'charcodes.dart'; |
| import 'constants.dart'; |
| import 'entry.dart'; |
| import 'format.dart'; |
| import 'header.dart'; |
| import 'utils.dart'; |
| |
| final class _WritingTransformer |
| extends StreamTransformerBase<TarEntry, List<int>> { |
| final OutputFormat format; |
| |
| const _WritingTransformer(this.format); |
| |
| @override |
| Stream<List<int>> bind(Stream<TarEntry> stream) { |
| // sync because the controller proxies another stream |
| final controller = StreamController<List<int>>(sync: true); |
| controller.onListen = () { |
| // Can be unawaited since it's the only thing done in onListen and since |
| // pipe is a terminal operation managing the remaining lifecycle of this |
| // stream controller. |
| unawaited(stream.pipe(tarWritingSink(controller, format: format))); |
| }; |
| |
| return controller.stream; |
| } |
| } |
| |
| /// A stream transformer writing tar entries as byte streams. |
| /// |
| /// Regardless of the input stream, the stream returned by this |
| /// [StreamTransformer.bind] is a single-subscription stream. |
| /// Apart from that, subscriptions, cancellations, pauses and resumes are |
| /// propagated as one would expect from a [StreamTransformer]. |
| /// |
| /// When piping the resulting stream into a [StreamConsumer], consider using |
| /// [tarWritingSink] directly. |
| /// To change the output format of files with long names, use [tarWriterWith]. |
| const StreamTransformer<TarEntry, List<int>> tarWriter = |
| _WritingTransformer(OutputFormat.pax); |
| |
| /// Creates a stream transformer writing tar entries as byte streams, with |
| /// custom encoding options. |
| /// |
| /// The [format] [OutputFormat] can be used to select the way tar entries with |
| /// long file or link names are written. By default, the writer will emit an |
| /// extended PAX header for the file ([OutputFormat.pax]). |
| /// Alternatively, [OutputFormat.gnuLongName] can be used to emit special tar |
| /// entries with the [TypeFlag.gnuLongName] type. |
| /// |
| /// Regardless of the input stream, the stream returned by this |
| /// [StreamTransformer.bind] is a single-subscription stream. |
| /// Apart from that, subscriptions, cancellations, pauses and resumes are |
| /// propagated as one would expect from a [StreamTransformer]. |
| /// |
| /// When using the default options, prefer using the constant [tarWriter] |
| /// instead. |
| StreamTransformer<TarEntry, List<int>> tarWriterWith( |
| {OutputFormat format = OutputFormat.pax}) { |
| return _WritingTransformer(format); |
| } |
| |
| /// Create a sink emitting encoded tar files to the [output] sink. |
| /// |
| /// For instance, you can use this to write a tar file: |
| /// |
| /// ```dart |
| /// import 'dart:convert'; |
| /// import 'dart:io'; |
| /// import 'package:tar/tar.dart'; |
| /// |
| /// Future<void> main() async { |
| /// Stream<TarEntry> entries = Stream.value( |
| /// TarEntry.data( |
| /// TarHeader( |
| /// name: 'example.txt', |
| /// mode: int.parse('644', radix: 8), |
| /// ), |
| /// utf8.encode('This is the content of the tar file'), |
| /// ), |
| /// ); |
| /// |
| /// final output = File('/tmp/test.tar').openWrite(); |
| /// await entries.pipe(tarWritingSink(output)); |
| /// } |
| /// ``` |
| /// |
| /// Note that, if you don't set the [TarHeader.size], outgoing tar entries need |
| /// to be buffered once, which decreases performance. |
| /// |
| /// The [format] argument can be used to control how long file names are written |
| /// in the tar archive. For more details, see the options in [OutputFormat]. |
| /// |
| /// See also: |
| /// - [tarWriter], a stream transformer using this sink |
| /// - [StreamSink] |
| StreamSink<TarEntry> tarWritingSink(StreamSink<List<int>> output, |
| {OutputFormat format = OutputFormat.pax}) { |
| return _WritingSink(output, format); |
| } |
| |
| /// A synchronous encoder for in-memory tar files. |
| /// |
| /// The default [tarWriter] creates an asynchronous conversion from a stream of |
| /// tar entries to a byte stream. |
| /// When all tar entries are in-memory ([SynchronousTarEntry]), it is possible |
| /// to write them synchronously too. |
| /// |
| /// To create a tar archive consisting of a single entry, use |
| /// [Converter.convert] on this [tarConverter]. |
| /// To create a tar archive consisting of any number of entries, first call |
| /// [Converter.startChunkedConversion] with a suitable output sink. Next, call |
| /// [Sink.add] for each tar entry and finish the archive by calling |
| /// [Sink.close]. |
| /// |
| /// To change the output format of the tar converter, use [tarConverterWith]. |
| /// To encode any kind of tar entries, use the asynchronous [tarWriter]. |
| const Converter<SynchronousTarEntry, List<int>> tarConverter = |
| _SynchronousTarConverter(OutputFormat.pax); |
| |
| /// A synchronous encoder for in-memory tar files, with custom encoding options. |
| /// |
| /// For more information on how to use the converter, see [tarConverter]. |
| Converter<SynchronousTarEntry, List<int>> tarConverterWith( |
| {OutputFormat format = OutputFormat.pax}) { |
| return _SynchronousTarConverter(format); |
| } |
| |
| /// This option controls how long file and link names should be written. |
| /// |
| /// This option can be passed to writer in [tarWritingSink] or[tarWriterWith]. |
| enum OutputFormat { |
| /// Generates an extended PAX headers to encode files with a long name. |
| /// |
| /// This is the default option. |
| pax, |
| |
| /// Generates [TypeFlag.gnuLongName] or [TypeFlag.gnuLongLink] entries when |
| /// encoding files with a long name. |
| /// |
| /// When this option is set, `package:tar` will not emit PAX headers which |
| /// may improve compatibility with some legacy systems like old 7zip versions. |
| /// |
| /// Note that this format can't encode large file sizes or long user names. |
| /// Tar entries can't be written if |
| /// * their [TarHeader.userName] is longer than 31 bytes in utf8, |
| /// * their [TarHeader.groupName] is longer than 31 bytes in utf8, or, |
| /// * their [TarEntry.contents] are larger than 8589934591 byte (around |
| /// 8 GiB). |
| /// |
| /// Attempting to encode such file will throw an [UnsupportedError]. |
| gnuLongName, |
| } |
| |
| final class _WritingSink implements StreamSink<TarEntry> { |
| final StreamSink<List<int>> _output; |
| final _SynchronousTarSink _synchronousWriter; |
| bool _closed = false; |
| final Completer<Object?> _done = Completer(); |
| |
| int _pendingOperations = 0; |
| Future<void> _ready = Future.value(); |
| |
| _WritingSink(this._output, OutputFormat format) |
| : _synchronousWriter = _SynchronousTarSink(_output, format); |
| |
| @override |
| Future<void> get done => _done.future; |
| |
| @override |
| Future<void> add(TarEntry event) { |
| if (_closed) { |
| throw StateError('Cannot add event after close was called'); |
| } |
| return _doWork(() => _safeAdd(event)); |
| } |
| |
| Future<void> _doWork(FutureOr<void> Function() work) { |
| _pendingOperations++; |
| // Chain futures to make sure we only write one entry at a time. |
| return _ready = _ready |
| .then((_) => work()) |
| .catchError(_output.addError) |
| .whenComplete(() { |
| _pendingOperations--; |
| |
| if (_closed && _pendingOperations == 0) { |
| _done.complete(_output.close()); |
| } |
| }); |
| } |
| |
| Future<void> _safeAdd(TarEntry event) async { |
| final header = event.header; |
| var size = header.size; |
| Uint8List? bufferedData; |
| if (size < 0) { |
| final builder = BytesBuilder(); |
| await event.contents.forEach(builder.add); |
| bufferedData = builder.takeBytes(); |
| size = bufferedData.length; |
| } |
| |
| _synchronousWriter._writeHeader(header, size); |
| |
| // Write content. |
| if (bufferedData != null) { |
| _output.add(bufferedData); |
| } else { |
| await _output.addStream(event.contents); |
| } |
| |
| _output.add(_paddingBytes(size)); |
| } |
| |
| @override |
| void addError(Object error, [StackTrace? stackTrace]) { |
| _output.addError(error, stackTrace); |
| } |
| |
| @override |
| Future<void> addStream(Stream<TarEntry> stream) async { |
| await for (final entry in stream) { |
| await add(entry); |
| } |
| } |
| |
| @override |
| Future<void> close() async { |
| if (!_closed) { |
| _closed = true; |
| |
| // Add two empty blocks at the end. |
| await _doWork(_synchronousWriter.close); |
| } |
| |
| return done; |
| } |
| } |
| |
| Uint8List _paddingBytes(int size) { |
| final padding = -size % blockSize; |
| assert((size + padding) % blockSize == 0 && |
| padding <= blockSize && |
| padding >= 0); |
| |
| return Uint8List(padding); |
| } |
| |
| final class _SynchronousTarConverter |
| extends Converter<SynchronousTarEntry, List<int>> { |
| final OutputFormat format; |
| |
| const _SynchronousTarConverter(this.format); |
| |
| @override |
| Sink<SynchronousTarEntry> startChunkedConversion(Sink<List<int>> sink) { |
| return _SynchronousTarSink(sink, format); |
| } |
| |
| @override |
| List<int> convert(SynchronousTarEntry input) { |
| final output = BytesBuilder(copy: false); |
| startChunkedConversion(ByteConversionSink.withCallback(output.add)) |
| ..add(input) |
| ..close(); |
| |
| return output.takeBytes(); |
| } |
| } |
| |
| final class _SynchronousTarSink implements Sink<SynchronousTarEntry> { |
| final OutputFormat _format; |
| final Sink<List<int>> _output; |
| |
| bool _closed = false; |
| int _paxHeaderCount = 0; |
| |
| _SynchronousTarSink(this._output, this._format); |
| |
| @override |
| void add(SynchronousTarEntry data) { |
| addHeaderAndData(data.header, data.data); |
| } |
| |
| void addHeaderAndData(TarHeader header, List<int> data) { |
| _throwIfClosed(); |
| |
| _writeHeader(header, data.length); |
| _output |
| ..add(data) |
| ..add(_paddingBytes(data.length)); |
| } |
| |
| @override |
| void close() { |
| if (_closed) return; |
| |
| // End the tar archive by writing two zero blocks. |
| _output |
| ..add(zeroBlock) |
| ..add(zeroBlock); |
| _output.close(); |
| |
| _closed = true; |
| } |
| |
| void _throwIfClosed() { |
| if (_closed) { |
| throw StateError('Encoder is closed. ' |
| 'After calling `endOfArchive()`, encoder must not be used.'); |
| } |
| } |
| |
| void _writeHeader(TarHeader header, int size) { |
| assert(header.size < 0 || header.size == size); |
| |
| var nameBytes = utf8.encode(header.name); |
| var linkBytes = utf8.encode(header.linkName ?? ''); |
| var gnameBytes = utf8.encode(header.groupName ?? ''); |
| var unameBytes = utf8.encode(header.userName ?? ''); |
| |
| // We only get 100 chars for the name and link name. If they are longer, we |
| // have to insert an entry just to store the names. Some tar implementations |
| // expect them to be zero-terminated, so use 99 chars to be safe. |
| final paxHeader = <String, List<int>>{}; |
| |
| if (nameBytes.length > 99) { |
| paxHeader[paxPath] = nameBytes; |
| nameBytes = nameBytes.sublist(0, 99); |
| } |
| if (linkBytes.length > 99) { |
| paxHeader[paxLinkpath] = linkBytes; |
| linkBytes = linkBytes.sublist(0, 99); |
| } |
| |
| // It's even worse for users and groups, where we only get 31 usable chars. |
| if (gnameBytes.length > 31) { |
| paxHeader[paxGname] = gnameBytes; |
| gnameBytes = gnameBytes.sublist(0, 31); |
| } |
| if (unameBytes.length > 31) { |
| paxHeader[paxUname] = unameBytes; |
| unameBytes = unameBytes.sublist(0, 31); |
| } |
| |
| if (size > maxIntFor12CharOct) { |
| paxHeader[paxSize] = ascii.encode(size.toString()); |
| } |
| |
| if (paxHeader.isNotEmpty) { |
| if (_format == OutputFormat.pax) { |
| _writePaxHeader(paxHeader); |
| } else { |
| _writeGnuLongName(paxHeader); |
| } |
| } |
| |
| final headerBlock = Uint8List(blockSize) |
| ..setAll(0, nameBytes) |
| ..setUint(header.mode, 100, 8) |
| ..setUint(header.userId, 108, 8) |
| ..setUint(header.groupId, 116, 8) |
| ..setUint(size, 124, 12) |
| ..setUint(header.modified.millisecondsSinceEpoch ~/ 1000, 136, 12) |
| ..[156] = typeflagToByte(header.typeFlag) |
| ..setAll(157, linkBytes) |
| ..setAll(257, magicUstar) |
| ..setUint(0, 263, 2) // version |
| ..setAll(265, unameBytes) |
| ..setAll(297, gnameBytes) |
| // To calculate the checksum, we first fill the checksum range with spaces |
| ..setAll(148, List.filled(8, $space)); |
| |
| // Then, we take the sum of the header |
| var checksum = 0; |
| for (final byte in headerBlock) { |
| checksum += byte; |
| } |
| headerBlock.setUint(checksum, 148, 8); |
| _output.add(headerBlock); |
| } |
| |
| /// Encodes an extended pax header. |
| /// |
| /// https://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_03 |
| void _writePaxHeader(Map<String, List<int>> values) { |
| final buffer = BytesBuilder(); |
| // format of each entry: "%d %s=%s\n", <length>, <keyword>, <value> |
| // note that the length includes the trailing \n and the length description |
| // itself. |
| values.forEach((key, value) { |
| final encodedKey = utf8.encode(key); |
| // +3 for the whitespace, the equals and the \n |
| final payloadLength = encodedKey.length + value.length + 3; |
| var indicatedLength = payloadLength; |
| |
| // The indicated length contains the length (in decimals) itself. So if |
| // we had payloadLength=9, then we'd prefix a 9 at which point the whole |
| // string would have a length of 10. If that happens, increment length. |
| var actualLength = payloadLength + indicatedLength.toString().length; |
| |
| while (actualLength != indicatedLength) { |
| indicatedLength++; |
| actualLength = payloadLength + indicatedLength.toString().length; |
| } |
| |
| // With that sorted out, let's add the line |
| buffer |
| ..add(utf8.encode(indicatedLength.toString())) |
| ..addByte($space) |
| ..add(encodedKey) |
| ..addByte($equal) |
| ..add(value) |
| ..addByte($lf); // \n |
| }); |
| |
| final paxData = buffer.takeBytes(); |
| addHeaderAndData( |
| HeaderImpl.internal( |
| format: TarFormat.pax, |
| modified: millisecondsSinceEpoch(0), |
| name: 'PaxHeader/${_paxHeaderCount++}', |
| mode: 0, |
| size: paxData.length, |
| typeFlag: TypeFlag.xHeader, |
| ), |
| paxData, |
| ); |
| } |
| |
| void _writeGnuLongName(Map<String, List<int>> values) { |
| // Ensure that a file that can't be written in the GNU format is not written |
| const allowedKeys = {paxPath, paxLinkpath}; |
| final invalidOptions = values.keys.toSet()..removeAll(allowedKeys); |
| if (invalidOptions.isNotEmpty) { |
| throw UnsupportedError( |
| 'Unsupported entry for OutputFormat.gnu. It uses long fields that ' |
| "can't be represented: $invalidOptions. \n" |
| 'Try using OutputFormat.pax instead.', |
| ); |
| } |
| |
| final name = values[paxPath]; |
| final linkName = values[paxLinkpath]; |
| |
| void create(List<int> name, TypeFlag flag) { |
| return addHeaderAndData( |
| HeaderImpl.internal( |
| name: '././@LongLink', |
| modified: millisecondsSinceEpoch(0), |
| format: TarFormat.gnu, |
| typeFlag: flag, |
| ), |
| name, |
| ); |
| } |
| |
| if (name != null) { |
| create(name, TypeFlag.gnuLongName); |
| } |
| if (linkName != null) { |
| create(linkName, TypeFlag.gnuLongLink); |
| } |
| } |
| } |
| |
| extension on Uint8List { |
| void setUint(int value, int position, int length) { |
| // Values are encoded as octal string, terminated and left-padded with |
| // space chars. |
| |
| // Set terminating space char. |
| this[position + length - 1] = $space; |
| |
| // Write as octal value, we write from right to left |
| var number = value; |
| var needsExplicitZero = number == 0; |
| |
| for (var pos = position + length - 2; pos >= position; pos--) { |
| if (number != 0) { |
| // Write the last octal digit of the number (e.g. the last 4 bits) |
| this[pos] = (number & 7) + $0; |
| // then drop the last digit (divide by 8 = 2³) |
| number >>= 3; |
| } else if (needsExplicitZero) { |
| this[pos] = $0; |
| needsExplicitZero = false; |
| } else { |
| // done, left-pad with spaces |
| this[pos] = $space; |
| } |
| } |
| } |
| } |