Make magic-byte detection allow a variable-size gap. A byte above 0xFF in the numbers list makes the matching try to continue at each of the next `byte-0xFF` positions. (It searches efficiently for the following byte, then recursively checks the rest of the pattern if finding that byte.) Add magic-byte recognition for `video/webv` and `video/x-matroska`, using this format, and remove the match for `audio/weba` (which is an extension for `audio/webm`, which isn't disitinguishable from `video/webm` without checking whether it contains any non-audio streams.)
diff --git a/pkgs/mime/CHANGELOG.md b/pkgs/mime/CHANGELOG.md index df0d6dc..427062b 100644 --- a/pkgs/mime/CHANGELOG.md +++ b/pkgs/mime/CHANGELOG.md
@@ -82,6 +82,15 @@ - `video/vnd.uvvu.mp4`, `uvvu` => `uvu` - `video/x-ms-asf`, `asx` => `asf` +* Allow magic byte patterns to have variable width gaps. + This allows distinguishing, for example, `video/x-matroska` and `video/webm` + files which have the same first four bytes, and are distinguished by a + document ID entry that does not have a fixed position. + The pattern entry limits how far ahead it can look. + +* Recognize `video/webm` and `video/x-matroska` by magic numbers, + and no longer detects all EBMF files as `audio/weba`. + ## 2.0.0 * **[Breaking]** `extensionFromMime(String mimeType)` returns `null` instead of @@ -105,7 +114,7 @@ ## 1.0.4 -* Changed `.js` to `text/javascript` per +* Changed `.js` to `text/javascript` per https://datatracker.ietf.org/doc/html/rfc9239. * Added `.mjs` as `text/javascript`. * Add `application/dicom` mimeType lookup by extension.
diff --git a/pkgs/mime/lib/src/magic_number.dart b/pkgs/mime/lib/src/magic_number.dart index a9cfd57..70868ed 100644 --- a/pkgs/mime/lib/src/magic_number.dart +++ b/pkgs/mime/lib/src/magic_number.dart
@@ -2,29 +2,71 @@ // for details. All rights reserved. Use of this source code is governed by a // BSD-style license that can be found in the LICENSE file. -const int initialMagicNumbersMaxLength = 12; +const int initialMagicNumbersMaxLength = 79; class MagicNumber { final String mimeType; + /// String containing code points in the 0..255 range to match. + /// + /// If a string contains a value above 255 (0xff), it's treated as + /// a size-limited lookahead wildcard. + /// A value of `0x110` will look for the following bytes starting + /// in the next 17 (0x11, value - 0xFF) characters. + /// If a mask is provided, it must have the same value at that position. final String numbers; + /// Optional string containing code points in the 0..255 range to mask with. final String? masks; - const MagicNumber(this.mimeType, this.numbers, [this.masks]); + const MagicNumber(this.mimeType, this.numbers, [this.masks]) + : assert(numbers.length > 0), + assert(masks == null || masks.length == numbers.length); bool matches(List<int> header) { - if (header.length < numbers.length) return false; + bool recursiveMatch(int byteCursor, int patternCursor) { + final masks = this.masks; + var mask = 0xFF; + while (patternCursor < numbers.length) { + if (byteCursor == header.length) return false; + var number = numbers.codeUnitAt(patternCursor); + if (masks != null) mask = masks.codeUnitAt(patternCursor); + if (number <= 0xFF) { + if ((number ^ header[byteCursor]) & mask != 0) { + return false; + } + patternCursor++; + byteCursor++; + } else { + // Wildcard spacer. + var lookaheadLength = 0; + do { + // Make sure masks has same value at that position. + if (masks != null && mask != number) return false; + lookaheadLength += number - 0xFF; + patternCursor++; + if (patternCursor == numbers.length) return true; + number = numbers.codeUnitAt(patternCursor); + if (masks != null) mask = masks.codeUnitAt(patternCursor); + // It's unnecessary to have multiple wildcards in a row, + // or having them at the end, but it'll work. + } while (number > 0xFF); - for (var i = 0; i < numbers.length; i++) { - final number = numbers.codeUnitAt(i); - if (masks != null) { - final mask = masks!.codeUnitAt(i); - if ((mask & number) != (mask & header[i])) return false; - } else { - if (number != header[i]) return false; + if (header.length < byteCursor + lookaheadLength) { + lookaheadLength = header.length - byteCursor; + } + for (var i = 0; i < lookaheadLength; i++) { + // Quick scan for first byte to match, before recursing. + if ((number ^ header[byteCursor + i]) & mask == 0 && + recursiveMatch(byteCursor + i + 1, patternCursor + 1)) { + return true; + } + } + return false; + } } + return true; } - return true; + return recursiveMatch(0, 0); } } @@ -65,7 +107,6 @@ MagicNumber('image/tiff', '\x4D\x4D\x00\x2A'), MagicNumber('audio/aac', '\xFF\xF1'), MagicNumber('audio/aac', '\xFF\xF9'), - MagicNumber('audio/weba', '\x1A\x45\xDF\xA3'), MagicNumber('audio/mpeg', '\x49\x44\x33'), MagicNumber('audio/mpeg', '\xFF\xFB'), MagicNumber('audio/ogg', '\x4F\x70\x75'), @@ -99,6 +140,10 @@ '\x00\x00\x00\x00\x66\x74\x79\x70\x6D\x70\x34\x32', '\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF', ), + // Look for EBMF DocType header within the next 64 bytes. + MagicNumber('video/webm', '\x1A\x45\xDF\xA3\u013f\x42\x82\x84webm'), + MagicNumber('video/x-matroska', '\x1A\x45\xDF\xA3\u013f\x42\x82\x88matroska'), + MagicNumber('model/gltf-binary', '\x46\x54\x6C\x67'), /// The WebP file format is based on the RIFF document format. @@ -143,3 +188,18 @@ '\x00\x00\x00\x00\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF', ), ]; + + +/// The maximum number of bytes matched by a numbers string. +/// +/// Accounts for wildcard matches. +/// +/// Do not export from public libraries. +int matchLength(String numbers) { + var length = numbers.length; + for (var i = 0; i < numbers.length; i++) { + final number = numbers.codeUnitAt(i); + if (number > 0xFF) length += number - 0x100; + } + return length; +}
diff --git a/pkgs/mime/lib/src/mime_type.dart b/pkgs/mime/lib/src/mime_type.dart index 5019eb5..e6b5aa3 100644 --- a/pkgs/mime/lib/src/mime_type.dart +++ b/pkgs/mime/lib/src/mime_type.dart
@@ -85,12 +85,14 @@ /// selective bits. The [mask] must have the same length as [bytes]. void addMagicNumber(List<int> bytes, String mimeType, {List<int>? mask}) { if (mask != null && bytes.length != mask.length) { - throw ArgumentError('Bytes and mask are of different lengths'); + throw ArgumentError('Bytes and mask are of different lengths', 'mask'); } - if (bytes.length > _magicNumbersMaxLength) { - _magicNumbersMaxLength = bytes.length; + final numbersString = String.fromCharCodes(bytes); + final length = matchLength(numbersString); + if (length > _magicNumbersMaxLength) { + _magicNumbersMaxLength = length; } - _magicNumbers.add(MagicNumber(mimeType, String.fromCharCodes(bytes), + _magicNumbers.add(MagicNumber(mimeType, numbersString, mask == null ? null : String.fromCharCodes(mask))); }
diff --git a/pkgs/mime/test/mime_type_test.dart b/pkgs/mime/test/mime_type_test.dart index bf30bc8..cf5879b 100644 --- a/pkgs/mime/test/mime_type_test.dart +++ b/pkgs/mime/test/mime_type_test.dart
@@ -279,6 +279,19 @@ 0x31, 0x00 ]); + _expectMimeType('file', 'video/webm', headerBytes: [ + 0x1A, 0x45, 0xDF, 0xA3, // EBMF header + 0xAF, 0x00, 0x00, 0x42, 0x00, // Anything + 0x42, 0x82, 0x84, ..."webm".codeUnits, // WebM DocID + 0x42, // Anything + ]); + _expectMimeType('file', 'video/x-matroska', headerBytes: [ + 0x1A, 0x45, 0xDF, 0xA3, // EBMF header + 0xAF, 0x00, 0x00, 0x42, 0x00, // Anything, up to 63 chars + 0x42, 0x82, 0x88, ..."matroska".codeUnits, // Matroska DocID + 0x42, // Anything + ]); + }); }); @@ -311,7 +324,7 @@ test('default magic number', () { final actualMaxBytes = initialMagicNumbers.fold<int>( 0, - (previous, magic) => math.max(previous, magic.numbers.length), + (previous, magic) => math.max(previous, matchLength(magic.numbers)), ); expect(initialMagicNumbersMaxLength, actualMaxBytes);