diff --git a/.gitignore b/.gitignore index d32ed3da..2706bb51 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,6 @@ target/ /debian/librofd-ffi0/ /debian/librofd-ffi-dev/ .zcode/ + +# Vendored dependencies for sw_64 offline builds (large, arch-specific) +vendor/ diff --git a/Cargo.lock b/Cargo.lock index 55b93e78..37bb7f6e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -107,9 +107,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "block-buffer" @@ -169,7 +169,7 @@ version = "0.20.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91e3bd0f4e25afa9cabc157908d14eeef9067d6448c49414d17b3fb55f0eadd0" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cairo-sys-rs", "freetype-rs", "glib", @@ -189,9 +189,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.4" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ad534f4357a5264cce5019c989cf66a4f0dc4e0d1b1d15f8aacec0ff7360273" +checksum = "a3eb0f42d6c360dc3f8a821f6bf2fdea7f72bfd36b3076eb0e6d1e9e0752fff4" dependencies = [ "find-msvc-tools", "jobserver", @@ -201,9 +201,9 @@ dependencies = [ [[package]] name = "cfg-expr" -version = "0.20.9" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4ece8474b5f766c63426647e7b4b316b67431ade1036a8313cee24a03ae917" +checksum = "1a2b34126159980f92da2a08bdec0694fd80fb5eb9e48aff25d20a0d8dfa710d" dependencies = [ "smallvec", "target-lexicon", @@ -310,9 +310,9 @@ dependencies = [ [[package]] name = "crc" -version = "3.4.0" +version = "3.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5eb8a2a1cd12ab0d987a5d5e825195d372001a4094a0376319d5a0ad71c1ba0d" +checksum = "9710d3b3739c2e349eb44fe848ad0b7c8cb1e42bd87ee49371df2f7acaf3e675" dependencies = [ "crc-catalog", ] @@ -325,24 +325,18 @@ checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" [[package]] name = "crc32fast" -version = "1.5.1" +version = "1.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8498c871161e1742aaa9d52551b2d6ebdd4c3d45a3be423e3728f33b955be550" +checksum = "01a7799fd6b852db0e61728dde9a204c423b44d689dbd432522543614b490e78" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" - -[[package]] -name = "crunchy" -version = "0.2.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-common" @@ -357,8 +351,6 @@ dependencies = [ [[package]] name = "deflate64" version = "0.1.12" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ac6b926516df9c60bfa16e107b21086399f8285a44ca9711344b9e553c5146e2" [[package]] name = "defmt" @@ -393,9 +385,12 @@ dependencies = [ [[package]] name = "deranged" -version = "0.5.8" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] [[package]] name = "derive_arbitrary" @@ -427,7 +422,7 @@ checksum = "c6232dd377dcc64799954cbd3a9bb882e9cdc1308ccd87b1c098f1fb2eaf82a8" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -459,12 +454,6 @@ version = "1.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" -[[package]] -name = "fax" -version = "0.2.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" - [[package]] name = "fdeflate" version = "0.3.7" @@ -476,9 +465,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "3e0f1c7c3a72c66fd80abe965175f7523475c0489a87d3ff9d6e8c87d87a9d2d" [[package]] name = "flate2" @@ -520,7 +509,7 @@ version = "0.37.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d1d1f81b925f09d7040682dbc91eb1b6ad43232f4bc6ee080f518001c05b5415" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "freetype-sys", "libc", ] @@ -570,7 +559,7 @@ checksum = "9fb9654ba8355388abeb8dcb4fc62f511300867002afc858860463bdd9fe0c44" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -611,27 +600,16 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "r-efi 5.3.0", + "r-efi", "wasip2", "wasm-bindgen", ] -[[package]] -name = "getrandom" -version = "0.4.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "300e883d756b2e4ec94e02791f39b04b522276138852cfc41d9fb7e904106099" -dependencies = [ - "cfg-if", - "libc", - "r-efi 6.0.0", -] - [[package]] name = "gif" -version = "0.14.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +checksum = "4ae047235e33e2829703574b54fdec96bfbad892062d97fed2f76022287de61b" dependencies = [ "color_quant", "weezl", @@ -656,7 +634,7 @@ version = "0.20.12" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ffc4b6e352d4716d84d7dde562dd9aee2a7d48beb872dd9ece7f2d1515b2d683" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "futures-channel", "futures-core", "futures-executor", @@ -705,22 +683,11 @@ dependencies = [ "system-deps", ] -[[package]] -name = "half" -version = "2.7.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" -dependencies = [ - "cfg-if", - "crunchy", - "zerocopy", -] - [[package]] name = "hashbrown" -version = "0.17.1" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed5909b6e89a2db4456e54cd5f673791d7eca6732202bbf2a9cc504fe2f9b84a" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" [[package]] name = "heck" @@ -739,15 +706,14 @@ dependencies = [ [[package]] name = "image" -version = "0.25.10" +version = "0.25.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +checksum = "db35664ce6b9810857a38a906215e75a9c879f0696556a39f59c62829710251a" dependencies = [ "bytemuck", "byteorder-lite", "color_quant", "gif", - "moxcms", "num-traits", "png", "tiff", @@ -757,9 +723,9 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.14.1" +version = "2.11.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07aa2048142242915a31d35844fb311e0e53fcca590c3a0a40dcf1b841fa09eb" +checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5" dependencies = [ "equivalent", "hashbrown", @@ -788,9 +754,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +checksum = "0ab1baf72f08796de0260609515130699b890ac25f30e610ad894bc5856cafdb" dependencies = [ "defmt", "jiff-core", @@ -803,18 +769,19 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +checksum = "5e52fe76043ccecc9005d2305ebaadf7d7fc0cc89ca6baa10a94d6bc68c7128c" dependencies = [ "defmt", + "log", ] [[package]] name = "jiff-static" -version = "0.2.35" +version = "0.2.37" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +checksum = "378268a1116ad67ae6228701118ac9f491d78fda38a40a1f1a9e1348de6f7212" dependencies = [ "jiff-core", "proc-macro2", @@ -824,19 +791,25 @@ dependencies = [ [[package]] name = "jobserver" -version = "0.1.35" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c00acbd29eabad4a2392fa0e921c874934dbbf4194312ad20f04a0ed67a3cb3" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.4.3", + "getrandom", "libc", ] +[[package]] +name = "jpeg-decoder" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00810f1d8b74be64b13dbf3db89ac67740615d6c891f0e7b6179326533011a07" + [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "ce57d20d1ea864ce2ac172ab472d409214f4fd359f0b2a2775abdf522e2af99e" dependencies = [ "cfg-if", "wasm-bindgen", @@ -922,21 +895,11 @@ dependencies = [ "simd-adler32", ] -[[package]] -name = "moxcms" -version = "0.8.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" -dependencies = [ - "num-traits", - "pxfm", -] - [[package]] name = "num-conv" -version = "0.2.2" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" [[package]] name = "num-traits" @@ -993,11 +956,11 @@ checksum = "f6b464fbc74e149a392436b17d523f769e057cb6877f6a5c4618bc6f11800548" [[package]] name = "png" -version = "0.18.1" +version = "0.17.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +checksum = "82151a2fc869e011c153adc57cf2789ccb8d9906ce52c0b39a6b5697749d7526" dependencies = [ - "bitflags 2.13.1", + "bitflags 1.3.2", "crc32fast", "fdeflate", "flate2", @@ -1012,9 +975,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] @@ -1027,9 +990,9 @@ checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" [[package]] name = "proc-macro-crate" -version = "3.5.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ "toml_edit", ] @@ -1043,12 +1006,6 @@ dependencies = [ "unicode-ident", ] -[[package]] -name = "pxfm" -version = "0.1.30" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d55d956fa96f5ec02be2e13af0e20391a5aa83d6a074e3ad368959d0fab299ea" - [[package]] name = "qmetaobject" version = "0.2.10" @@ -1086,12 +1043,6 @@ dependencies = [ "semver", ] -[[package]] -name = "quick-error" -version = "2.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" - [[package]] name = "quote" version = "1.0.47" @@ -1107,12 +1058,6 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" -[[package]] -name = "r-efi" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" - [[package]] name = "regex" version = "1.13.1" @@ -1251,7 +1196,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] @@ -1269,9 +1214,9 @@ dependencies = [ [[package]] name = "serde_spanned" -version = "1.1.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6662b5879511e06e8999a8a235d848113e942c9124f211511b16466ee2995f26" +checksum = "f8bbf91e5a4d6315eee45e704372590b30e260ee83af6639d64557f51b067776" dependencies = [ "serde_core", ] @@ -1316,9 +1261,9 @@ dependencies = [ [[package]] name = "smallvec" -version = "1.16.0" +version = "1.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9be42f50aa861c555654aa3a37f52f4b1074bacf4e48fe0ef7fa584e80f1f0f" +checksum = "ba467056f1b547ed52077911161fc86985becbc60e8e1857c8a144dab0def891" [[package]] name = "subtle" @@ -1350,9 +1295,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "12df2e0110f65b775f769bb17ef989067a1d931b2eb822bd4346631eeada89f9" dependencies = [ "proc-macro2", "quote", @@ -1374,9 +1319,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.5" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "adb6935a6f5c20170eeceb1a3835a49e12e19d792f6dd344ccc76a985ca5a6ca" +checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" [[package]] name = "thiserror" @@ -1415,107 +1360,104 @@ checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.5", ] [[package]] name = "tiff" -version = "0.11.3" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +checksum = "ba1310fcea54c6a9a4fd1aad794ecc02c31682f6bfbecdf460bf19533eed1e3e" dependencies = [ - "fax", "flate2", - "half", - "quick-error", + "jpeg-decoder", "weezl", - "zune-jpeg", ] [[package]] name = "time" -version = "0.3.55" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cdb87b95ec50ddfa440816d227a17b2ccbdda963a316a727fda0fc4334f7d134" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", "num-conv", "powerfmt", - "serde_core", + "serde", "time-core", ] [[package]] name = "time-core" -version = "0.1.9" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e1c906769ad99c88eaa54e728060edef082f8e358ff32030cb7c7d315e81109" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" [[package]] name = "tinyvec" -version = "1.13.2" +version = "1.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4cf0ded5c4e56918d8f8a339e1bb67d038d3bc6d144ac407904015ba2e4cde9b" -dependencies = [ - "tinyvec_macros", -] - -[[package]] -name = "tinyvec_macros" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +checksum = "fd3ca314f692efd6c868f8408f53fe444634a845f96c028b97d35f6a1f79f0ee" [[package]] name = "toml" -version = "1.1.5+spec-1.1.0" +version = "1.0.7+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12c0ba9680044b4ce98d391a62094047eada0d64860b80166c39f4a6b5640785" +checksum = "dd28d57d8a6f6e458bc0b8784f8fdcc4b99a437936056fa122cb234f18656a96" dependencies = [ "indexmap", "serde_core", "serde_spanned", - "toml_datetime", + "toml_datetime 1.0.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.7.5+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92e1cfed4a3038bc5a127e35a2d360f145e1f4b971b551a2ba5fd7aedf7e1347" +dependencies = [ + "serde_core", ] [[package]] name = "toml_datetime" -version = "1.1.1+spec-1.1.0" +version = "1.0.1+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3165f65f62e28e0115a00b2ebdd37eb6f3b641855f9d636d3cd4103767159ad7" +checksum = "9b320e741db58cac564e26c607d3cc1fdc4a88fd36c879568c07856ed83ff3e9" dependencies = [ "serde_core", ] [[package]] name = "toml_edit" -version = "0.25.13+spec-1.1.0" +version = "0.23.10+spec-1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" +checksum = "84c8b9f757e028cee9fa244aea147aab2a9ec09d5325a9b01e0a49730c2b5269" dependencies = [ "indexmap", - "toml_datetime", + "toml_datetime 0.7.5+spec-1.1.0", "toml_parser", - "winnow", + "winnow 0.7.15", ] [[package]] name = "toml_parser" -version = "1.1.3+spec-1.1.0" +version = "1.0.10+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" +checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420" dependencies = [ - "winnow", + "winnow 1.0.4", ] [[package]] name = "toml_writer" -version = "1.1.2+spec-1.1.0" +version = "1.0.7+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d56353a2a665ad0f41a421187180aab746c8c325620617ad883a99a1cbe66d2" +checksum = "f17aaa1c6e3dc22b1da4b6bba97d066e354c7945cac2f7852d4e4e7ca7a6b56d" [[package]] name = "ttf-parser" @@ -1573,9 +1515,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "aecb87a33d3b0c5e3b7aa46336eaf486cffafbd281b195e4c8b80d50df2351bf" dependencies = [ "cfg-if", "once_cell", @@ -1586,9 +1528,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "a690d511e3c1a8b3a55e33511e3c2c00c78415cd23650f32b808627f5696b9ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -1596,22 +1538,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "411e4887f0071ef2d2164a9d5fdf2d20efbef78fccd3a78b0c10a1dc5295e48a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.5", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.128" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "81941cd78d0c92026c33e5e01312845a4cb1e9af3407f9134b100dd03144103e" dependencies = [ "unicode-ident", ] @@ -1712,13 +1654,19 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "winnow" -version = "1.0.4" +version = "0.7.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" dependencies = [ "memchr", ] +[[package]] +name = "winnow" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23b97319f7b8343df12cc98938e5c3eb436064524c8d2b4e30a1d3a36eecdf81" + [[package]] name = "wit-bindgen" version = "0.57.1" @@ -1740,40 +1688,20 @@ dependencies = [ "lzma-sys", ] -[[package]] -name = "zerocopy" -version = "0.8.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5a105cd7b140f6eeec8acff2ea38135d3cab283ada58540f629fe51e46696eb" -dependencies = [ - "zerocopy-derive", -] - -[[package]] -name = "zerocopy-derive" -version = "0.8.55" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fe976fb70c78cd64cccfe3a6fc142244e8a77b70959b30faf9d0ac37ee228eb" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.119", -] - [[package]] name = "zeroize" -version = "1.9.0" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.5.0" +version = "1.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" +checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" dependencies = [ "proc-macro2", "quote", @@ -1795,7 +1723,7 @@ dependencies = [ "deflate64", "displaydoc", "flate2", - "getrandom 0.3.4", + "getrandom", "hmac", "indexmap", "lzma-rs", @@ -1845,18 +1773,18 @@ dependencies = [ [[package]] name = "zstd-safe" -version = "7.2.4" +version = "7.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +checksum = "64d80649ab6db9d9f6f9c80a40becd948eda4714a0a5ac8c4d157a32231c7882" dependencies = [ "zstd-sys", ] [[package]] name = "zstd-sys" -version = "2.0.16+zstd.1.5.7" +version = "2.1.0+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +checksum = "0ef0a8027ec3ee71300ab3bcbcd0393f434aa72b91ca6d635a39941deae8eea0" dependencies = [ "cc", "pkg-config", @@ -1864,15 +1792,15 @@ dependencies = [ [[package]] name = "zune-core" -version = "0.5.3" +version = "0.4.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d56377fd46368984a170bc5aac5567e52ca5da874caa60bea39fcbca78fb658b" +checksum = "3f423a2c17029964870cfaabb1f13dfab7d092a62a29a89264f4d36990ca414a" [[package]] name = "zune-jpeg" -version = "0.5.15" +version = "0.4.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +checksum = "29ce2c8a9384ad323cf564b67da86e21d3cfdff87908bc1223ed5c99bc792713" dependencies = [ "zune-core", ] diff --git a/Cargo.toml b/Cargo.toml index 8ace4340..0339454f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,4 +1,5 @@ [package] +rust-version.workspace = true name = "rofd" version = "0.4.0" edition = "2021" @@ -35,6 +36,7 @@ default-members = ["crates/rofd-core", "crates/rofd-render", "crates/rofd-ffi"] resolver = "2" [workspace.package] +rust-version = "1.81" edition = "2021" license = "LGPL-2.1-or-later" repository = "https://github.com/hualet/rofd" @@ -50,3 +52,8 @@ serde-xml-rs = "0.6.0" thiserror = "2.0.20" xml-rs = "0.8.27" zip = { version = "2.4.2", default-features = false, features = ["deflate"] } + +# Patch: replace nightly-only `unbounded_shr` with stable `checked_shr` +# (required for rustc 1.81 used on loongarch64/sw_64 build hosts) +[patch.crates-io] +deflate64 = { path = "patches/deflate64" } diff --git a/crates/rofd-core/Cargo.toml b/crates/rofd-core/Cargo.toml index 40ef1ef8..b519193c 100644 --- a/crates/rofd-core/Cargo.toml +++ b/crates/rofd-core/Cargo.toml @@ -2,6 +2,7 @@ name = "rofd-core" version = "0.4.0" edition.workspace = true +rust-version.workspace = true license.workspace = true repository.workspace = true description = "Safe OFD document parsing and query API" diff --git a/crates/rofd-core/src/paint.rs b/crates/rofd-core/src/paint.rs index d0cb24d1..2ad12312 100644 --- a/crates/rofd-core/src/paint.rs +++ b/crates/rofd-core/src/paint.rs @@ -241,7 +241,7 @@ impl Color { .map(parse_channel) .collect::>>()?; let space = match ColorSpaceKind::from_channel_count(channels.len()) { - Some(space) if !strict || declared_space.is_none_or(|declared| declared == space) => { + Some(space) if !strict || declared_space.map_or(true, |declared| declared == space) => { space } _ => { diff --git a/crates/rofd-ffi/Cargo.toml b/crates/rofd-ffi/Cargo.toml index 316f15a5..c5b65c6e 100644 --- a/crates/rofd-ffi/Cargo.toml +++ b/crates/rofd-ffi/Cargo.toml @@ -2,6 +2,7 @@ name = "rofd-ffi" version = "0.4.0" edition.workspace = true +rust-version.workspace = true license.workspace = true repository.workspace = true description = "Stable C ABI for the rofd OFD reader and renderer" diff --git a/crates/rofd-ffi/src/error.rs b/crates/rofd-ffi/src/error.rs index bc592648..7d930420 100644 --- a/crates/rofd-ffi/src/error.rs +++ b/crates/rofd-ffi/src/error.rs @@ -486,7 +486,7 @@ impl SlotRange { return Ok(None); } let start = pointer as usize; - if !start.is_multiple_of(align_of::()) { + if start % align_of::() != 0 { return Err(()); } let end = start.checked_add(size_of::()).ok_or(())?; @@ -502,7 +502,7 @@ impl SlotRange { return Ok(None); } let start = pointer as usize; - if !start.is_multiple_of(alignment) { + if start % alignment != 0 { return Err(()); } let end = start.checked_add(size).ok_or(())?; diff --git a/crates/rofd-render/Cargo.toml b/crates/rofd-render/Cargo.toml index 895a3eed..a1a2f928 100644 --- a/crates/rofd-render/Cargo.toml +++ b/crates/rofd-render/Cargo.toml @@ -2,6 +2,7 @@ name = "rofd-render" version = "0.4.0" edition.workspace = true +rust-version.workspace = true license.workspace = true repository.workspace = true description = "Backend-neutral OFD display-list lowering and Cairo rendering" diff --git a/crates/rofd-render/src/cairo_renderer.rs b/crates/rofd-render/src/cairo_renderer.rs index a685cf8b..74949987 100644 --- a/crates/rofd-render/src/cairo_renderer.rs +++ b/crates/rofd-render/src/cairo_renderer.rs @@ -1084,11 +1084,11 @@ impl RenderGeometry { || viewport .x .checked_add(viewport.width) - .is_none_or(|right| right > canvas.pixel_width) + .map_or(true, |right| right > canvas.pixel_width) || viewport .y .checked_add(viewport.height) - .is_none_or(|bottom| bottom > canvas.pixel_height) + .map_or(true, |bottom| bottom > canvas.pixel_height) { return Err(Error::InvalidOption { field: "viewport", diff --git a/patches/deflate64/.cargo-ok b/patches/deflate64/.cargo-ok new file mode 100644 index 00000000..5f8b7958 --- /dev/null +++ b/patches/deflate64/.cargo-ok @@ -0,0 +1 @@ +{"v":1} \ No newline at end of file diff --git a/patches/deflate64/.cargo_vcs_info.json b/patches/deflate64/.cargo_vcs_info.json new file mode 100644 index 00000000..f0bc8643 --- /dev/null +++ b/patches/deflate64/.cargo_vcs_info.json @@ -0,0 +1,6 @@ +{ + "git": { + "sha1": "a0dd6ef0ade9e0458a4c05bb9ea45f815f7ac62a" + }, + "path_in_vcs": "" +} \ No newline at end of file diff --git a/patches/deflate64/.git-blame-ignore-revs b/patches/deflate64/.git-blame-ignore-revs new file mode 100644 index 00000000..d471fe40 --- /dev/null +++ b/patches/deflate64/.git-blame-ignore-revs @@ -0,0 +1,2 @@ +# cargo fmt +766b6d314e3c1208f8275d7c83c1deab0030d857 diff --git a/patches/deflate64/.gitignore b/patches/deflate64/.gitignore new file mode 100644 index 00000000..fa8d85ac --- /dev/null +++ b/patches/deflate64/.gitignore @@ -0,0 +1,2 @@ +Cargo.lock +target diff --git a/patches/deflate64/CHANGELOG.md b/patches/deflate64/CHANGELOG.md new file mode 100644 index 00000000..446877f8 --- /dev/null +++ b/patches/deflate64/CHANGELOG.md @@ -0,0 +1,99 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog]. + +[Keep a Changelog]: https://keepachangelog.com/en/1.1.0/ + +## [Unreleased] +### Added + +### Changed + +### Deprecated + +### Removed + +### Fixed + +### Security + +## [0.1.12] - 2026-03-23 +### Fixed +- Unexpected errors with long window size [`#46`](https://github.com/anatawa12/deflate64-rs/pull/46) + +## [0.1.11] - 2026-02-26 +### Added +- Checkpoint feature for partial-progress restoration [`#40`](https://github.com/anatawa12/deflate64-rs/pull/40) +- test: `cargo bench` simple benchmark + +### Changed +- perf: optimize decoder inner loop for 2x performance [`#41`](https://github.com/anatawa12/deflate64-rs/pull/41) + +## [0.1.10] - 2025-10-01 +## [0.1.9] - 2024-07-16 +### Fixed +- Infinite loop with empty output buffer in `Deflate64Decoder` [`#30`](https://github.com/anatawa12/deflate64-rs/pull/30) + +## [0.1.8] - 2024-03-11 +### Fixed +- Panic with Invalid Data [`#26`](https://github.com/anatawa12/deflate64-rs/pull/26) [`#24`](https://github.com/anatawa12/deflate64-rs/pull/24) + +## [0.1.7] - 2024-01-27 +## [0.1.6] - 2023-10-15 +### Fixed +- Overflow substract with some archive [`#14`](https://github.com/anatawa12/deflate64-rs/pull/14) + +## [0.1.5] - 2023-08-19 +### Added +- test: 7zip compatibility test +- Changelog file + +### Changed +- Remove `unsafe` code + +## [0.1.4] +### Added +- `Deflate64Decoder`, Streaming `Read` decoder implementation + +### Fixed +- Overflow error in debug build + +## [0.1.3] +### Added +- Many documentation comment +- `InflaterManaged.errored()` + +### Changed +- Remove Box usage in `InflaterManaged` + +## [0.1.2] - 2023-01-16 +### Fixed +- Release build will cause compilation error + +### Fixed +- Several bugs + +## [0.1.1] - 2023-01-15 +### Added +- Implement Debug in many struct + +## [0.1.0] - 2023-07-29 +### Added +- Initial Deflate64 implementation + +[Unreleased]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.12...HEAD +[0.1.12]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.11...v0.1.12 +[0.1.11]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.10...v0.1.11 +[0.1.10]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.9...v0.1.10 +[0.1.9]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.8...v0.1.9 +[0.1.8]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.7...v0.1.8 +[0.1.7]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.6...v0.1.7 +[0.1.6]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.5...v0.1.6 +[0.1.5]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.4...v0.1.5 +[0.1.4]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.3...v0.1.4 +[0.1.3]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.2...v0.1.3 +[0.1.2]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.1...v0.1.2 +[0.1.1]: https://github.com/anatawa12/deflate64-rs/compare/v0.1.0...v0.1.1 +[0.1.0]: https://github.com/anatawa12/deflate64-rs/releases/tag/v0.1.0 diff --git a/patches/deflate64/Cargo.toml b/patches/deflate64/Cargo.toml new file mode 100644 index 00000000..6b634d9e --- /dev/null +++ b/patches/deflate64/Cargo.toml @@ -0,0 +1,70 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies. +# +# If you are reading this file be aware that the original Cargo.toml +# will likely look very different (and much more reasonable). +# See Cargo.toml.orig for the original contents. + +[package] +edition = "2021" +name = "deflate64" +version = "0.1.12" +authors = ["anatawa12 "] +build = false +exclude = [ + ".github/**", + "tests/**", + "test-assets/**", +] +autolib = false +autobins = false +autoexamples = false +autotests = false +autobenches = false +description = "Deflate64 implementation based on .NET's implementation" +homepage = "https://github.com/anatawa12/deflate64-rs#readme" +readme = "README.md" +keywords = [ + "deflate", + "deflate64", + "compression", +] +categories = ["compression"] +license = "MIT" +repository = "https://github.com/anatawa12/deflate64-rs" + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = [ + "--cfg", + "docsrs", +] + +[features] +checkpoint = [] +default = [] + +[lib] +name = "deflate64" +path = "src/lib.rs" + +[[bench]] +name = "inflate" +path = "benches/inflate.rs" +harness = false + +[dependencies] + +[dev-dependencies.bytemuck] +version = "1.13.1" +features = ["derive"] + +[dev-dependencies.proptest] +version = "1.2.0" + +[dev-dependencies.tempfile] +version = "3.7.1" diff --git a/patches/deflate64/Cargo.toml.orig b/patches/deflate64/Cargo.toml.orig new file mode 100644 index 00000000..95a59ccc --- /dev/null +++ b/patches/deflate64/Cargo.toml.orig @@ -0,0 +1,38 @@ +[package] +name = "deflate64" +version = "0.1.12" +edition = "2021" + +license = "MIT" +authors = ["anatawa12 "] +description = "Deflate64 implementation based on .NET's implementation" +homepage = "https://github.com/anatawa12/deflate64-rs#readme" +repository = "https://github.com/anatawa12/deflate64-rs" +readme = "README.md" +keywords = ["deflate", "deflate64", "compression"] +categories = ["compression"] + +exclude = [ + ".github/**", + "tests/**", + "test-assets/**", +] + +[features] +default = [] +checkpoint = [] + +[dependencies] + +[dev-dependencies] +bytemuck = { version = "1.13.1", features = ["derive"] } +proptest = "1.2.0" +tempfile = "3.7.1" + +[[bench]] +name = "inflate" +harness = false + +[package.metadata.docs.rs] +all-features = true +rustdoc-args = ["--cfg", "docsrs"] diff --git a/patches/deflate64/LICENSE b/patches/deflate64/LICENSE new file mode 100644 index 00000000..84698d3d --- /dev/null +++ b/patches/deflate64/LICENSE @@ -0,0 +1,24 @@ +The MIT License (MIT) + +Copyright (c) .NET Foundation and Contributors +Copyright (c) anatawa12 2023 + +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/patches/deflate64/README.md b/patches/deflate64/README.md new file mode 100644 index 00000000..8766f0bb --- /dev/null +++ b/patches/deflate64/README.md @@ -0,0 +1,5 @@ +Deflate64 implementation based on [.NET's implementation][dotnet-impl] + +This is made to unzip zip file with deflate64 made with windows 11. + +[dotnet-impl]: https://github.com/dotnet/runtime/tree/e5efd8010e19593298dc2c3ee15106d5aec5a924/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateManaged diff --git a/patches/deflate64/benches/inflate.rs b/patches/deflate64/benches/inflate.rs new file mode 100644 index 00000000..fcfae97d --- /dev/null +++ b/patches/deflate64/benches/inflate.rs @@ -0,0 +1,33 @@ +use deflate64::InflaterManaged; +use std::hint::black_box; +use std::time::Instant; + +static ZIP_FILE_DATA: &[u8] = include_bytes!("../test-assets/deflate64.zip"); +const BINARY_WAV_DATA_OFFSET: usize = 40; +const BINARY_WAV_COMPRESSED_SIZE: usize = 2669743; +const BINARY_WAV_UNCOMPRESSED_SIZE: usize = 2703788; +const ITERATIONS: usize = 150; + +fn main() { + let compressed = &ZIP_FILE_DATA[BINARY_WAV_DATA_OFFSET..][..BINARY_WAV_COMPRESSED_SIZE]; + let mut output = vec![0u8; BINARY_WAV_UNCOMPRESSED_SIZE + 10]; + + let start = Instant::now(); + for _ in 0..ITERATIONS { + let mut inflater = InflaterManaged::new(); + let result = black_box(inflater.inflate(black_box(compressed), &mut output)); + assert_eq!(result.bytes_written, BINARY_WAV_UNCOMPRESSED_SIZE); + } + let elapsed = start.elapsed(); + + let ms_per_iter = elapsed.as_secs_f64() * 1000.0 / ITERATIONS as f64; + let mb_per_sec = + (BINARY_WAV_UNCOMPRESSED_SIZE * ITERATIONS) as f64 / elapsed.as_secs_f64() / 1_000_000.0; + + println!(); + println!( + "benchmark complete - {:.2} ms/iter, {:.1} MB/s", + ms_per_iter, mb_per_sec + ); + println!(); +} diff --git a/patches/deflate64/src/buffer.rs b/patches/deflate64/src/buffer.rs new file mode 100644 index 00000000..e39f9d23 --- /dev/null +++ b/patches/deflate64/src/buffer.rs @@ -0,0 +1,59 @@ +use std::mem::MaybeUninit; +use std::ops::{IndexMut, RangeBounds}; + +pub(crate) enum Buffer<'a> { + Uninit(&'a mut [MaybeUninit]), + Init(&'a mut [u8]), +} + +impl<'b> Buffer<'b> { + #[inline(always)] + pub(crate) fn reborrow<'m>(&'m mut self) -> Buffer<'m> + where + 'b: 'm, + { + match &mut *self { + Self::Uninit(uninit) => Buffer::Uninit(&mut uninit[..]), + Self::Init(init) => Buffer::Init(&mut init[..]), + } + } + + #[inline(always)] + pub(crate) fn index_mut(self, range: impl RangeBounds) -> Self { + let range = (range.start_bound().cloned(), range.end_bound().cloned()); + + match self { + Buffer::Uninit(uninit) => Buffer::Uninit(uninit.index_mut(range)), + Buffer::Init(init) => Buffer::Init(init.index_mut(range)), + } + } + + pub(crate) fn copy_from_slice(&mut self, input: &[u8]) { + match self { + Buffer::Uninit(uninit) => { + // TODO: replace with write_copy_of_slice when rust-lang/rust#79995 get stabilized. + debug_assert_eq!(uninit.len(), input.len()); + uninit + .iter_mut() + .zip(input) + .for_each(|(maybe_uninit, byte)| { + maybe_uninit.write(*byte); + }); + } + Buffer::Init(init) => init.copy_from_slice(input), + }; + } + + #[inline(always)] + pub(crate) fn len(&self) -> usize { + match self { + Buffer::Uninit(uninit) => uninit.len(), + Buffer::Init(init) => init.len(), + } + } + + #[inline(always)] + pub(crate) fn is_empty(&self) -> bool { + self.len() == 0 + } +} diff --git a/patches/deflate64/src/huffman_tree.rs b/patches/deflate64/src/huffman_tree.rs new file mode 100644 index 00000000..91dc1e38 --- /dev/null +++ b/patches/deflate64/src/huffman_tree.rs @@ -0,0 +1,319 @@ +use crate::input_buffer::InputBuffer; +use crate::InternalErr; + +// Packing: bits 0-8 = symbol (0-288), bits 9-13 = code length (1-16), bits 14+ = zero +const SYMBOL_BITS: u8 = 9; +const SYMBOL_MASK: i16 = (1 << SYMBOL_BITS) - 1; // 0x1FF + +fn pack(symbol: i16, code_len: u8) -> i16 { + symbol | ((code_len as i16) << SYMBOL_BITS) +} + +pub(crate) fn unpack(entry: i16) -> (u16, i32) { + ((entry & SYMBOL_MASK) as u16, (entry >> SYMBOL_BITS) as i32) +} + +#[derive(Debug)] +pub(crate) struct HuffmanTree { + code_lengths_length: u16, + table: [i16; 1 << Self::TABLE_BITS], + // Table stores positive or negative numbers. Positive numbers are packed symbols + // and code lengths (see pack/unpack above). Negative values are indexes into a + // binary tree of array nodes; consume additional bits for left/right navagation + // until a positive packed value is reached. Note, the original implementation had + // separate "left" and "right" tables, we have interleaved these tables to enable + // branchless left/right navigation with simple math. Left and right nodes come in + // pairs, where N*2 is a left node and N*2+1 is a right node. + nodes: [i16; Self::MAX_CODE_LENGTHS * 4], + code_length_array: [u8; Self::MAX_CODE_LENGTHS], +} + +impl HuffmanTree { + pub(crate) const MAX_CODE_LENGTHS: usize = 288; + pub(crate) const TABLE_BITS: u8 = 9; + pub(crate) const TABLE_BITS_MASK: usize = (1 << Self::TABLE_BITS) - 1; + + pub(crate) const MAX_LITERAL_TREE_ELEMENTS: usize = 288; + pub(crate) const MAX_DIST_TREE_ELEMENTS: usize = 32; + pub(crate) const END_OF_BLOCK_CODE: usize = 256; + pub(crate) const NUMBER_OF_CODE_LENGTH_TREE_ELEMENTS: usize = 19; + + pub fn invalid() -> Self { + HuffmanTree { + code_lengths_length: Default::default(), + table: [0i16; 1 << Self::TABLE_BITS], + nodes: [0i16; Self::MAX_CODE_LENGTHS * 4], + code_length_array: [0u8; Self::MAX_CODE_LENGTHS], + } + } + + pub fn static_literal_length_tree() -> Self { + HuffmanTree::new(&Self::get_static_literal_tree_length()).unwrap() + } + + pub fn static_distance_tree() -> Self { + HuffmanTree::new(&Self::get_static_distance_tree_length()).unwrap() + } + + fn assert_code_lengths_len(len: usize) { + debug_assert!( + len == Self::MAX_LITERAL_TREE_ELEMENTS + || len == Self::MAX_DIST_TREE_ELEMENTS + || len == Self::NUMBER_OF_CODE_LENGTH_TREE_ELEMENTS, + "we only expect three kinds of Length here" + ); + } + + pub fn new(code_lengths: &[u8]) -> Result { + Self::assert_code_lengths_len(code_lengths.len()); + let code_lengths_length = code_lengths.len(); + + // I need to find proof that left and right array will always be + // enough. I think they are. + + let mut instance = Self { + table: [0; 1 << Self::TABLE_BITS], + nodes: [0; Self::MAX_CODE_LENGTHS * 4], + code_lengths_length: code_lengths_length as u16, + code_length_array: { + let mut buffer = [0u8; Self::MAX_CODE_LENGTHS]; + buffer[..code_lengths.len()].copy_from_slice(code_lengths); + buffer + }, + }; + + instance.create_table()?; + + Ok(instance) + } + + pub fn new_in_place(&mut self, code_lengths: &[u8]) -> Result<(), InternalErr> { + Self::assert_code_lengths_len(code_lengths.len()); + self.table.fill(0); + self.nodes.fill(0); + self.code_lengths_length = code_lengths.len() as u16; + self.code_length_array[..code_lengths.len()].copy_from_slice(code_lengths); + self.code_length_array[code_lengths.len()..].fill(0); + + self.create_table() + } + + // Generate the array contains huffman codes lengths for static huffman tree. + // The data is in RFC 1951. + fn get_static_literal_tree_length() -> [u8; Self::MAX_LITERAL_TREE_ELEMENTS] { + let mut literal_tree_length = [0u8; Self::MAX_LITERAL_TREE_ELEMENTS]; + + literal_tree_length[0..][..144].fill(8); + literal_tree_length[144..][..112].fill(9); + literal_tree_length[256..][..24].fill(7); + literal_tree_length[280..][..8].fill(8); + literal_tree_length + } + + const fn get_static_distance_tree_length() -> [u8; Self::MAX_DIST_TREE_ELEMENTS] { + [5u8; Self::MAX_DIST_TREE_ELEMENTS] + } + + fn bit_reverse(code: u32, length: usize) -> u32 { + debug_assert!(length > 0 && length <= 16, "Invalid len"); + code.reverse_bits() >> (32 - length) + } + + fn calculate_huffman_code(&self) -> [u32; Self::MAX_LITERAL_TREE_ELEMENTS] { + let code_lengths = &self.code_length_array[..self.code_lengths_length as usize]; + let mut bit_length_count = [0u32; 17]; + for &code_length in code_lengths.iter() { + bit_length_count[code_length as usize] += 1; + } + bit_length_count[0] = 0; // clear count for length 0 + + let mut next_code = [0u32; 17]; + let mut temp_code = 0u32; + + for bits in 1..=16 { + temp_code = (temp_code + bit_length_count[bits - 1]) << 1; + next_code[bits] = temp_code; + } + + let mut code = [0u32; Self::MAX_LITERAL_TREE_ELEMENTS]; + for (i, &len) in code_lengths.iter().enumerate() { + if len > 0 { + code[i] = Self::bit_reverse(next_code[len as usize], len as usize); + next_code[len as usize] += 1; + } + } + + code + } + + fn create_table(&mut self) -> Result<(), InternalErr> { + let code_array = self.calculate_huffman_code(); + let code_lengths_len = self.code_lengths_length as usize; + + let mut avail = 1; // skip 0 because -0 is still 0, can't distinguish by sign + + for (ch, &len) in self.code_length_array[..code_lengths_len] + .iter() + .enumerate() + { + if len > 0 { + // start value (bit reversed) + let mut start = code_array[ch] as usize; + + if len <= Self::TABLE_BITS { + // If a particular symbol is shorter than nine bits, + // then that symbol's translation is duplicated + // in all those entries that start with that symbol's bits. + // For example, if the symbol is four bits, then it's duplicated + // 32 times in a nine-bit table. If a symbol is nine bits long, + // it appears in the table once. + // + // Make sure that in the loop below, code is always + // less than table_size. + // + // On last iteration we store at array index: + // initial_start_at + (locs-1)*increment + // = initial_start_at + locs*increment - increment + // = initial_start_at + (1 << tableBits) - increment + // = initial_start_at + table_size - increment + // + // Therefore we must ensure: + // initial_start_at + table_size - increment < table_size + // or: initial_start_at < increment + // + let increment = 1 << len; + if start >= increment { + return Err(InternalErr::DataError); // InvalidHuffmanData + } + + // Note the bits in the table are reverted. + let locs = 1 << (Self::TABLE_BITS - len); + for _ in 0..locs { + self.table[start] = pack(ch as i16, len); + start += increment; + } + } else { + // For any code which has length longer than num_elements, + // build a binary tree. + + let mut overflow_bits = len - Self::TABLE_BITS; // the nodes we need to represent the data. + let mut code_bit_mask = 1 << Self::TABLE_BITS; // mask to get current bit (the bits can't fit in the table) + + // the left, right table is used to represent the + // the rest bits. When we got the first part (number bits.) and look at + // tbe table, we will need to follow the tree to find the real character. + // This is in place to avoid bloating the table if there are + // a few ones with long code. + // As an optimization, we now store left/right together at N*2 and N*2+1. + // We store (-left_index) as a pointer to newly allocated node pairs; the + // get_symbol logic increments the negated left_index to get right_index. + let mut index = start & Self::TABLE_BITS_MASK; + let mut value: &mut i16 = &mut self.table[index]; + + while { + if *value == 0 { + // set up next pointer if this node is not used before. + // store -left_index directly (avail * 2) + *value = -(avail * 2); + avail += 1; + } + + if *value > 0 { + // prevent an IndexOutOfRangeException from array[index] + return Err(InternalErr::DataError); // InvalidHuffmanData + } + + // left child at -value, right child at -value+1 + let left_index = (-*value) as usize; + index = left_index + ((start & code_bit_mask) != 0) as usize; + + value = self.nodes.get_mut(index).ok_or(InternalErr::DataError)?; // InvalidHuffmanData + + code_bit_mask <<= 1; + overflow_bits -= 1; + + overflow_bits != 0 + } {} + + *value = pack(ch as i16, len); + } + } + } + + Ok(()) + } + + pub fn get_next_symbol(&self, input: &mut InputBuffer<'_>) -> Result { + debug_assert_ne!(self.code_lengths_length, 0, "invalid table"); + // Try to load 16 bits into input buffer if possible and get the bit_buffer value. + // If there aren't 16 bits available we will return all we have in the + // input buffer. + let bit_buffer = input.try_load_16bits(); + if input.available_bits() == 0 { + // running out of input. + return Err(InternalErr::DataNeeded); + } + + // decode an element + let mut entry = self.table[bit_buffer as usize & Self::TABLE_BITS_MASK]; + let mut bits = bit_buffer >> Self::TABLE_BITS; + while entry < 0 { + // navigate the tree: left child at -entry, right at -entry+1 + let child_index = ((-entry) as usize) + (bits & 1) as usize; + entry = self.nodes[child_index]; + // shift bits down and mask for branchless left/right indexing + bits >>= 1; + } + + let (symbol, code_length) = unpack(entry); + + if code_length <= 0 || code_length > 16 { + return Err(InternalErr::DataError); // InvalidHuffmanData + } + + // If this code is longer than the # bits we had in the bit buffer (i.e. + // we read only part of the code), we can hit the entry in the table or the tree + // for another symbol. However the length of another symbol will not match the + // available bits count. + if code_length > input.available_bits() { + // We already tried to load 16 bits and maximum length is 15, + // so this means we are running out of input. + return Err(InternalErr::DataNeeded); + } + + input.skip_bits(code_length); + Ok(symbol) + } + + // get_next_symbol_assume_input is an optimization of get_next_symbol when the caller + // knows that 16 bits exist in the bit buffer or are available as input bytes. It is + // meant for use in an optimized decode loop that strictly verifies this precondition. + // If the precondition is violated, the call will assert in debug builds and is likely + // to produce an incorrect symbol in release builds. + #[inline(always)] + pub fn get_next_symbol_assume_input( + &self, + input: &mut InputBuffer<'_>, + ) -> Result { + debug_assert_ne!(self.code_lengths_length, 0, "invalid table"); + let bit_buffer = input.load_16bits_assume_input(); + let mut entry = self.table[bit_buffer as usize & Self::TABLE_BITS_MASK]; + let mut bits = bit_buffer >> Self::TABLE_BITS; + while entry < 0 { + let child_index = ((-entry) as usize) + (bits & 1) as usize; + entry = self.nodes[child_index]; + bits >>= 1; + } + let (symbol, code_length) = unpack(entry); + if code_length == 0 { + return Err(InternalErr::DataError); + } + input.skip_bits(code_length); + Ok(symbol) + } + + #[allow(dead_code)] + pub fn code_lengths(&self) -> &[u8] { + &self.code_length_array[..self.code_lengths_length as usize] + } +} diff --git a/patches/deflate64/src/inflater_checkpoint.rs b/patches/deflate64/src/inflater_checkpoint.rs new file mode 100644 index 00000000..bde6860c --- /dev/null +++ b/patches/deflate64/src/inflater_checkpoint.rs @@ -0,0 +1,346 @@ +//! Checkpoint support for saving and restoring partial decompression progress. +//! +//! This module provides [`checkpoint()`](super::InflaterManaged::checkpoint) and +//! [`restore_from_checkpoint()`](super::InflaterManaged::restore_from_checkpoint) +//! methods on [`InflaterManaged`](super::InflaterManaged) for persisting inflater +//! state across process restarts. This is useful when decompressing very large +//! streams where a crash or interruption would otherwise lose all progress. +//! +//! # Stability +//! +//! The checkpoint format is experimental and may change between library versions. +//! Checkpoints include an internal version number; `restore_from_checkpoint()` +//! returns `None` for incompatible versions. Do not rely on checkpoints persisting +//! across library upgrades. +//! +//! # Usage +//! +//! Checkpoints are typically saved periodically during decompression: +//! +//! ```ignore +//! let mut bytes_written = 0u64; +//! loop { +//! let n = input.read(&mut input_buf)?; +//! if n == 0 && inflater.finished() { break; } +//! +//! let result = inflater.inflate(&input_buf[..n], &mut output_buf); +//! output.write_all(&output_buf[..result.bytes_written])?; +//! bytes_written += result.bytes_written as u64; +//! +//! // Save checkpoint every 100 MB +//! if bytes_written % 100_000_000 < result.bytes_written as u64 { +//! if let Some((data, _)) = inflater.checkpoint() { +//! std::fs::write("checkpoint.dat", &data)?; +//! } +//! } +//! } +//! ``` +//! +//! To restore from a checkpoint, call `restore_from_checkpoint()` and seek both +//! streams to the positions indicated by the returned [`CheckpointStreamPositions`](CheckpointStreamPositions): +//! +//! ```ignore +//! let mut inflater = InflaterManaged::new(); +//! if let Some(pos) = inflater.restore_from_checkpoint(&checkpoint_data) { +//! input.seek(SeekFrom::Start(pos.input_bytes_to_skip))?; +//! output.seek(SeekFrom::Start(pos.output_bytes_already_returned))?; +//! // Continue with normal inflate() loop +//! } +//! ``` +//! +//! # Security +//! +//! Checkpoint data represents internal program state. While `restore_from_checkpoint()` +//! validates checksums and structural integrity, malformed data from untrusted sources +//! could cause decompression errors or incorrect output. Only restore checkpoints you +//! created. +//! +//! # Checkpoint Size +//! +//! Checkpoints are typically around 65KB but can reach 131KB if the inflater has +//! significant buffered output not yet drained by the caller. The format consists +//! of a 346-byte header (version, bit position, block state, Huffman code lengths, +//! output counters), followed by the output window history, and a Fletcher-32 +//! checksum. + +use crate::huffman_tree::HuffmanTree; +use crate::input_buffer::{BitsBuffer, InputBuffer}; +use crate::{BlockType, InflaterState}; + +use super::{InflaterManaged, TABLE_LOOKUP_DISTANCE_MAX}; + +// Checkpoint binary format (little-endian): +// +// Offset Size Field +// ------ ---- ---------------------------------- +// 0 2 version: u16 (currently 0x1001) +// 2 8 input_bits: u64 +// 10 1 buffered_value: u8 (0-7 unread bits) +// 11 1 bfinal_block_type: u8 ((bfinal << 7) | block_type) +// 12 2 uncompressed_remaining: u16 +// 14 288 lit_code_lengths: [u8; 288] +// 302 32 dist_code_lengths: [u8; 32] +// 334 8 output_bytes_written: u64 +// 342 4 output_bytes_unread: u32 +// 346 var window_data: [u8] (len = max(min(65538, output_bytes_written), output_bytes_unread)) +// END-4 4 checksum: u32 (Fletcher-32) + +const CHECKPOINT_HEADER_SIZE: usize = 346; + +fn fletcher32_checksum(data: &[u8]) -> u32 { + let (mut a, mut b) = (0u32, 0u32); + for &byte in data { + a = a.wrapping_add(byte as u32); + b = b.wrapping_add(a); + } + (b << 16) | (a & 0xFFFF) +} + +/// Update checkpoint state after a write to the output window or end-of-block. +/// Called from the parent module's decode functions. +#[inline(always)] +pub(super) fn update_checkpoint( + inflater: &mut InflaterManaged, + input: &InputBuffer<'_>, + end_of_block: bool, +) { + debug_assert!(input.available_bits() >= 0 && input.available_bits() <= 32); + // checkpoint_input_bits tracks the number of input bits consumed up to the checkpoint. + inflater.checkpoint_input_bits = + (inflater.total_input_loaded + input.read_bytes as u64) * 8 - input.available_bits() as u64; + // checkpoint_bit_buffer holds unconsumed bits of the most recently loaded input byte. + inflater.checkpoint_bit_buffer = input.peek_available_bits() as u8; + // checkpoint_bfinal_block_type holds bfinal state and current block type. + // End-of-block is stored as uncompressed with zero remaining (functionally identical). + let bfinal_flag = (inflater.bfinal as u8) << 7; + if end_of_block { + debug_assert!(matches!( + inflater.state, + InflaterState::ReadingBFinal | InflaterState::Done + )); + inflater.checkpoint_bfinal_block_type = BlockType::Uncompressed as u8 | bfinal_flag; + } else { + match inflater.block_type { + BlockType::Uncompressed => { + debug_assert_eq!(inflater.state, InflaterState::DecodingUncompressed); + debug_assert!(inflater.block_length > 0); + } + BlockType::Static => debug_assert_eq!(inflater.state, InflaterState::DecodeTop), + BlockType::Dynamic => debug_assert_eq!(inflater.state, InflaterState::DecodeTop), + }; + inflater.checkpoint_bfinal_block_type = inflater.block_type as u8 | bfinal_flag; + } +} + +impl InflaterManaged { + /// Serialize the most recent inflater checkpoint for use with + /// [`restore_from_checkpoint()`](Self::restore_from_checkpoint). + /// + /// Returns `None` if no checkpoint is available (no data processed yet, + /// inflater errored, or decompression already complete with output drained). + /// + /// The returned checkpoint can contain up to 129KB of data representing the + /// inflater state and history buffer. The `CheckpointStreamPositions` describes + /// the input/output byte offsets corresponding to this checkpoint. + #[cfg_attr(docsrs, doc(cfg(feature = "checkpoint")))] + pub fn checkpoint(&self) -> Option<(Vec, CheckpointStreamPositions)> { + if self.checkpoint_input_bits == 0 + || self.errored() + || (self.output.available_bytes() == 0 && self.state == InflaterState::Done) + { + return None; + } + + let checkpoint_block_type = + BlockType::from_int((self.checkpoint_bfinal_block_type & 0x7F) as u16)?; + let uncompressed_remaining = match checkpoint_block_type { + BlockType::Uncompressed => self.block_length as u32, + _ => 0, + }; + + let mut lit_codes = [0; HuffmanTree::MAX_LITERAL_TREE_ELEMENTS]; + let mut dist_codes = [0; HuffmanTree::MAX_DIST_TREE_ELEMENTS]; + if checkpoint_block_type == BlockType::Dynamic { + let lens = self.literal_length_tree.code_lengths(); + lit_codes[..lens.len()].copy_from_slice(lens); + let lens = self.distance_tree.code_lengths(); + dist_codes[..lens.len()].copy_from_slice(lens); + } + + let output_bytes_written = + self.total_output_consumed + self.output.available_bytes() as u64; + let bytes_unread = self.output.available_bytes() as u32; + let (window_a, window_b) = self.output.get_checkpoint_data(output_bytes_written); + + let bfinal_block_type = self.checkpoint_bfinal_block_type; + + // Mask unreferenced high bits for deterministic serialization + let num_buffered_bits = (8 - (self.checkpoint_input_bits & 7)) as u32 & 7; + let buffered_value = self.checkpoint_bit_buffer & ((1 << num_buffered_bits) - 1); + + let mut out = Vec::with_capacity(CHECKPOINT_HEADER_SIZE + window_a.len() + window_b.len()); + out.extend_from_slice(&0x1001u16.to_le_bytes()); // version + out.extend_from_slice(&self.checkpoint_input_bits.to_le_bytes()); + out.push(buffered_value); + out.push(bfinal_block_type); + out.extend_from_slice(&(uncompressed_remaining as u16).to_le_bytes()); + out.extend_from_slice(&lit_codes); + out.extend_from_slice(&dist_codes); + out.extend_from_slice(&output_bytes_written.to_le_bytes()); + out.extend_from_slice(&bytes_unread.to_le_bytes()); + debug_assert_eq!(out.len(), CHECKPOINT_HEADER_SIZE); + out.extend_from_slice(window_a); + out.extend_from_slice(window_b); + let checksum = fletcher32_checksum(&out); + out.extend_from_slice(&checksum.to_le_bytes()); + + Some(( + out, + CheckpointStreamPositions { + input_bytes_to_skip: self.checkpoint_input_bits.div_ceil(8), + output_bytes_already_returned: output_bytes_written - bytes_unread as u64, + }, + )) + } + + /// Restore inflater state from a previously serialized checkpoint. + /// + /// Returns `None` if the data is corrupt, invalid, or from an incompatible + /// library version. On success, the inflater's internal state is overwritten + /// and the caller must seek input/output streams according to the returned + /// `CheckpointStreamPositions`. + /// + /// If the inflater has an output byte limit from + /// [`with_uncompressed_size()`](Self::with_uncompressed_size), that limit is + /// retained and checkpoints exceeding it will not be restored. + #[cfg_attr(docsrs, doc(cfg(feature = "checkpoint")))] + #[must_use] + pub fn restore_from_checkpoint( + &mut self, + checkpoint_data: &[u8], + ) -> Option { + if checkpoint_data.len() < CHECKPOINT_HEADER_SIZE + 4 { + return None; + } + let (data, checksum_bytes) = checkpoint_data.split_at(checkpoint_data.len() - 4); + let stored_checksum = u32::from_le_bytes(checksum_bytes.try_into().ok()?); + if fletcher32_checksum(data) != stored_checksum { + return None; + } + + let mut cursor = data; + let mut read = |n: usize| -> Option<&[u8]> { + if cursor.len() < n { + return None; + } + let (head, tail) = cursor.split_at(n); + cursor = tail; + Some(head) + }; + + let version: u16 = u16::from_le_bytes(read(2)?.try_into().ok()?); + if version != 0x1001 { + return None; + } + let input_bits: u64 = u64::from_le_bytes(read(8)?.try_into().ok()?); + let buffered_value: u8 = read(1)?[0]; + let bfinal_block_type: u8 = read(1)?[0]; + let remaining_uncompressed: u16 = u16::from_le_bytes(read(2)?.try_into().ok()?); + let lit_codes: &[u8] = read(HuffmanTree::MAX_LITERAL_TREE_ELEMENTS)?; + let dist_codes: &[u8] = read(HuffmanTree::MAX_DIST_TREE_ELEMENTS)?; + let output_bytes_written: u64 = u64::from_le_bytes(read(8)?.try_into().ok()?); + let output_bytes_unread: u32 = u32::from_le_bytes(read(4)?.try_into().ok()?); + let window_data: &[u8] = cursor; + + let num_buffered_bits = (8 - (input_bits & 7)) as i32 & 7; + let bits = BitsBuffer::from_bits(buffered_value as u32, num_buffered_bits); + + let expected_window_len = (output_bytes_written.min(TABLE_LOOKUP_DISTANCE_MAX as u64) + as u32) + .max(output_bytes_unread) as usize; + if window_data.len() != expected_window_len + || window_data.len() > crate::output_window::WINDOW_SIZE + { + return None; + } + + let output_already_returned = output_bytes_written - output_bytes_unread as u64; + if self.uncompressed_size != usize::MAX + && output_already_returned > self.uncompressed_size as u64 + { + return None; + } + + let bfinal = (bfinal_block_type & 128) != 0; + let block_type = BlockType::from_int((bfinal_block_type % 128).into())?; + + let mut lit_tree = HuffmanTree::invalid(); + let mut dist_tree = HuffmanTree::invalid(); + if block_type == BlockType::Dynamic { + if lit_codes.iter().any(|x| *x > 16) || dist_codes.iter().any(|x| *x > 16) { + return None; + } + lit_tree.new_in_place(lit_codes).ok()?; + dist_tree.new_in_place(dist_codes).ok()?; + } else if block_type == BlockType::Uncompressed + && remaining_uncompressed > 0 + && bits.bits_in_buffer != 0 + { + return None; + } + + // All validation passed - modify self + self.bits = bits; + self.checkpoint_input_bits = input_bits; + self.checkpoint_bit_buffer = buffered_value; + self.total_output_consumed = output_bytes_written - output_bytes_unread as u64; + self.total_input_loaded = input_bits.div_ceil(8); + + self.output + .restore_from_checkpoint(window_data, output_bytes_unread as usize); + + self.checkpoint_bfinal_block_type = bfinal_block_type; + match block_type { + BlockType::Uncompressed => { + self.bfinal = bfinal; + self.block_type = BlockType::Uncompressed; + self.block_length = remaining_uncompressed as usize; + if remaining_uncompressed > 0 { + self.state = InflaterState::DecodingUncompressed; + } else if !bfinal { + self.state = InflaterState::ReadingBFinal; + } else { + self.state = InflaterState::Done; + } + } + BlockType::Static => { + self.bfinal = bfinal; + self.block_type = BlockType::Static; + self.literal_length_tree = HuffmanTree::static_literal_length_tree(); + self.distance_tree = HuffmanTree::static_distance_tree(); + self.state = InflaterState::DecodeTop; + } + BlockType::Dynamic => { + self.bfinal = bfinal; + self.block_type = BlockType::Dynamic; + self.literal_length_tree = lit_tree; + self.distance_tree = dist_tree; + self.state = InflaterState::DecodeTop; + } + } + + Some(CheckpointStreamPositions { + input_bytes_to_skip: input_bits.div_ceil(8), + output_bytes_already_returned: output_bytes_written - output_bytes_unread as u64, + }) + } +} + +/// Input and output stream positions corresponding to an inflater checkpoint. +#[derive(Debug, PartialEq, Eq)] +pub struct CheckpointStreamPositions { + /// Count of input bytes already consumed before checkpoint. + pub input_bytes_to_skip: u64, + /// Count of output bytes already returned before checkpoint. + pub output_bytes_already_returned: u64, +} diff --git a/patches/deflate64/src/inflater_managed.rs b/patches/deflate64/src/inflater_managed.rs new file mode 100644 index 00000000..954d113a --- /dev/null +++ b/patches/deflate64/src/inflater_managed.rs @@ -0,0 +1,799 @@ +use crate::buffer::Buffer; +use crate::huffman_tree::HuffmanTree; +use crate::input_buffer::{BitsBuffer, InputBuffer}; +use crate::output_window::OutputWindow; +use crate::{array_copy, array_copy1, BlockType, InflateResult, InflaterState, InternalErr}; +use std::cmp::min; +use std::mem::MaybeUninit; + +// Extra bits for length code 257 - 285. +static EXTRA_LENGTH_BITS: [u8; 29] = [ + 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 4, 4, 4, 4, 5, 5, 5, 5, 16, +]; + +// The base length for length code 257 - 285. +// The formula to get the real length for a length code is lengthBase[code - 257] + (value stored in extraBits) +static LENGTH_BASE: [u8; 29] = [ + 3, 4, 5, 6, 7, 8, 9, 10, 11, 13, 15, 17, 19, 23, 27, 31, 35, 43, 51, 59, 67, 83, 99, 115, 131, + 163, 195, 227, 3, +]; + +// The base distance for distance code 0 - 31 +// The real distance for a distance code is distanceBasePosition[code] + (value stored in extraBits) +static DISTANCE_BASE_POSITION: [u16; 32] = [ + 1, 2, 3, 4, 5, 7, 9, 13, 17, 25, 33, 49, 65, 97, 129, 193, 257, 385, 513, 769, 1025, 1537, + 2049, 3073, 4097, 6145, 8193, 12289, 16385, 24577, 32769, 49153, +]; + +// code lengths for code length alphabet is stored in following order +static CODE_ORDER: [u8; 19] = [ + 16, 17, 18, 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15, +]; + +static STATIC_DISTANCE_TREE_TABLE: [u8; 32] = [ + 0x00, 0x10, 0x08, 0x18, 0x04, 0x14, 0x0c, 0x1c, 0x02, 0x12, 0x0a, 0x1a, 0x06, 0x16, 0x0e, 0x1e, + 0x01, 0x11, 0x09, 0x19, 0x05, 0x15, 0x0d, 0x1d, 0x03, 0x13, 0x0b, 0x1b, 0x07, 0x17, 0x0f, 0x1f, +]; + +// source: https://github.com/dotnet/runtime/blob/82dac28143be0740d795f434db9b70f61b3b7a04/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateManaged/OutputWindow.cs#L17 +// Note: the upstream comment says "65536 length as well as up to a 65538 distance", but actually +// it looks limit is 65538 length with 65538 distance. +// There is no mention for length / distance limit in [APPNOTE.TXT] so it's hard to know +// which is correct, but 65538 length with 65538 distance is much reasonable since +// original deflate have (2^8-1)+3 length with 2^15 distance, and +// 65538 is (2^16-1)+3 and 65536 is 2^15. +// [APPNOTE.TXT]: https://pkware.cachefly.net/webdocs/casestudies/APPNOTE.TXT +const TABLE_LOOKUP_LENGTH_MAX: usize = 65538; +pub(crate) const TABLE_LOOKUP_DISTANCE_MAX: usize = 65536; + +/// The streaming Inflater for deflate64 +/// +/// This struct has big buffer so It's not recommended to move this struct. +#[derive(Debug)] +pub struct InflaterManaged { + output: OutputWindow, + bits: BitsBuffer, + literal_length_tree: HuffmanTree, + distance_tree: HuffmanTree, + + state: InflaterState, + bfinal: bool, + block_type: BlockType, + + // uncompressed block + block_length_buffer: [u8; 4], + block_length: usize, + + // compressed block + length: usize, + distance_code: u16, + extra_bits: i32, + + loop_counter: u32, + literal_length_code_count: u32, + distance_code_count: u32, + code_length_code_count: u32, + code_array_size: u32, + length_code: u16, + + code_list: [u8; HuffmanTree::MAX_LITERAL_TREE_ELEMENTS + HuffmanTree::MAX_DIST_TREE_ELEMENTS], // temporary array to store the code length for literal/Length and distance + code_length_tree_code_length: [u8; HuffmanTree::NUMBER_OF_CODE_LENGTH_TREE_ELEMENTS], + deflate64: bool, + code_length_tree: HuffmanTree, + uncompressed_size: usize, + + // Cumulative counters updated once per inflate call + total_input_loaded: u64, // total bytes loaded into bit reader, only updated after decode() + total_output_consumed: u64, // total bytes returned to caller (also used for uncompressed_size limit) + + // Lightweight checkpoint: updated after every write to output window + #[cfg(feature = "checkpoint")] + checkpoint_input_bits: u64, // exact input bit position of checkpoint + #[cfg(feature = "checkpoint")] + checkpoint_bit_buffer: u8, // low byte of input bit_buffer (future bits) + #[cfg(feature = "checkpoint")] + checkpoint_bfinal_block_type: u8, // (bfinal << 7) | block_type +} + +impl InflaterManaged { + /// Initializes Inflater + #[allow(clippy::new_without_default)] + #[inline] + pub fn new() -> Self { + Self::with_uncompressed_size(usize::MAX) + } + + /// Initializes Inflater with expected uncompressed size. + pub fn with_uncompressed_size(uncompressed_size: usize) -> Self { + Self { + output: OutputWindow::new(), + bits: BitsBuffer::new(), + + literal_length_tree: HuffmanTree::invalid(), + code_list: [0u8; HuffmanTree::MAX_LITERAL_TREE_ELEMENTS + + HuffmanTree::MAX_DIST_TREE_ELEMENTS], + code_length_tree_code_length: [0u8; HuffmanTree::NUMBER_OF_CODE_LENGTH_TREE_ELEMENTS], + deflate64: true, + code_length_tree: HuffmanTree::invalid(), + uncompressed_size, + state: InflaterState::ReadingBFinal, // start by reading BFinal bit + bfinal: false, + block_type: BlockType::Uncompressed, + block_length_buffer: [0u8; 4], + block_length: 0, + length: 0, + distance_code: 0, + extra_bits: 0, + loop_counter: 0, + literal_length_code_count: 0, + distance_code_count: 0, + code_length_code_count: 0, + code_array_size: 0, + distance_tree: HuffmanTree::invalid(), + length_code: 0, + total_input_loaded: 0, + total_output_consumed: 0, + #[cfg(feature = "checkpoint")] + checkpoint_input_bits: 0, + #[cfg(feature = "checkpoint")] + checkpoint_bit_buffer: 0, + #[cfg(feature = "checkpoint")] + checkpoint_bfinal_block_type: 0, + } + } + + /// Returns true if decompression finished and no more output is available + /// + /// This also returns true if this inflater is in error state + pub fn finished(&self) -> bool { + (self.state == InflaterState::Done && self.available_output() == 0) + || self.state == InflaterState::DataErrored + } + + /// Returns true if decompression finished, but may still have output available in buffer + /// + /// This also returns true if this inflater is in error state + pub fn input_finished(&self) -> bool { + self.state == InflaterState::Done || self.state == InflaterState::DataErrored + } + + /// Returns true if this inflater is in error state + pub fn errored(&self) -> bool { + self.state == InflaterState::DataErrored + } + + /// The count of bytes currently inflater has in internal output buffer + #[allow(dead_code)] + pub fn available_output(&self) -> usize { + self.output.available_bytes() + } + + /// Try to decompress from `input` to `output`. + /// + /// This will decompress data until `output` is full, `input` is empty, + /// the end if the deflate64 stream is hit, or there is error data in the deflate64 stream. + pub fn inflate(&mut self, input: &[u8], output: &mut [u8]) -> InflateResult { + self.inflate_internal(input, Buffer::Init(output)) + } + + /// Same as [`Self::inflate`] but accepts uninitialized buffer + pub fn inflate_uninit( + &mut self, + input: &[u8], + output: &mut [MaybeUninit], + ) -> InflateResult { + self.inflate_internal(input, Buffer::Uninit(output)) + } + + fn inflate_internal(&mut self, input: &[u8], mut output: Buffer<'_>) -> InflateResult { + // copy bytes from output to outputbytes if we have available bytes + // if buffer is not filled up. keep decoding until no input are available + // if decodeBlock returns false. Throw an exception. + let mut result = InflateResult::new(); + let mut input = InputBuffer::new(self.bits, input); + while 'while_loop: { + let mut copied = 0; + if self.uncompressed_size == usize::MAX { + copied = self.output.copy_to(output.reborrow()); + } else if (self.uncompressed_size as u64) > self.total_output_consumed { + let remaining = + (self.uncompressed_size as u64 - self.total_output_consumed) as usize; + let len = min(output.len(), remaining); + output = output.index_mut(..len); + copied = self.output.copy_to(output.reborrow()); + } else { + self.state = InflaterState::Done; + self.output.clear_bytes_used(); + } + if copied > 0 { + output = output.index_mut(copied..); + result.bytes_written += copied; + self.total_output_consumed += copied as u64; + } + + if output.is_empty() { + // filled in the bytes buffer + break 'while_loop false; + } + // decode will return false when more input is needed + if self.errored() { + result.data_error = true; + break 'while_loop false; + } else if self.input_finished() { + break 'while_loop false; + } + match self.decode(&mut input) { + Ok(()) => true, + Err(InternalErr::DataNeeded) => false, + Err(InternalErr::DataError) => { + self.state = InflaterState::DataErrored; + result.data_error = true; + false + } + } + } {} + + self.bits = input.bits; + self.total_input_loaded += input.read_bytes as u64; + result.bytes_consumed = input.read_bytes; + result + } + + fn decode(&mut self, input: &mut InputBuffer<'_>) -> Result<(), InternalErr> { + let mut eob = false; + let result; + + if self.errored() { + return Err(InternalErr::DataError); + } else if self.input_finished() { + return Ok(()); + } + + if self.state == InflaterState::ReadingBFinal { + // reading bfinal bit + // Need 1 bit + self.bfinal = input.get_bits(1)? != 0; + self.state = InflaterState::ReadingBType; + } + + if self.state == InflaterState::ReadingBType { + // Need 2 bits + self.state = InflaterState::ReadingBType; + let bits = input.get_bits(2)?; + + self.block_type = BlockType::from_int(bits).ok_or(InternalErr::DataError)?; + match self.block_type { + BlockType::Dynamic => { + self.state = InflaterState::ReadingNumLitCodes; + } + BlockType::Static => { + self.literal_length_tree = HuffmanTree::static_literal_length_tree(); + self.distance_tree = HuffmanTree::static_distance_tree(); + self.state = InflaterState::DecodeTop; + } + BlockType::Uncompressed => { + self.state = InflaterState::UncompressedAligning; + } + } + } + + if self.block_type == BlockType::Dynamic { + if self.state < InflaterState::DecodeTop { + // we are reading the header + result = self.decode_dynamic_block_header(input); + } else { + result = self.decode_block(input, &mut eob); // this can returns true when output is full + } + } else if self.block_type == BlockType::Static { + result = self.decode_block(input, &mut eob); + } else if self.block_type == BlockType::Uncompressed { + result = self.decode_uncompressed_block(input, &mut eob); + } else { + result = Err(InternalErr::DataError); // UnknownBlockType + } + + // + // If we reached the end of the block and the block we were decoding had + // bfinal=1 (final block) + // + if eob && self.bfinal { + self.state = InflaterState::Done; + } + result + } + + fn decode_uncompressed_block( + &mut self, + input: &mut InputBuffer<'_>, + end_of_block: &mut bool, + ) -> Result<(), InternalErr> { + *end_of_block = false; + loop { + match self.state { + InflaterState::UncompressedAligning => { + input.skip_to_byte_boundary(); + self.state = InflaterState::UncompressedByte1; + continue; //goto case InflaterState.UncompressedByte1; + } + InflaterState::UncompressedByte1 + | InflaterState::UncompressedByte2 + | InflaterState::UncompressedByte3 + | InflaterState::UncompressedByte4 => { + self.block_length_buffer + [(self.state - InflaterState::UncompressedByte1) as usize] = + input.get_bits(8)? as u8; + if self.state == InflaterState::UncompressedByte4 { + self.block_length = self.block_length_buffer[0] as usize + + (self.block_length_buffer[1] as usize) * 256; + let block_length_complement: i32 = self.block_length_buffer[2] as i32 + + (self.block_length_buffer[3] as i32) * 256; + + // make sure complement matches + if self.block_length as u16 != !block_length_complement as u16 { + return Err(InternalErr::DataError); // InvalidBlockLength + } + } + + self.state = match self.state { + InflaterState::UncompressedByte1 => InflaterState::UncompressedByte2, + InflaterState::UncompressedByte2 => InflaterState::UncompressedByte3, + InflaterState::UncompressedByte3 => InflaterState::UncompressedByte4, + InflaterState::UncompressedByte4 => InflaterState::DecodingUncompressed, + _ => unreachable!(), + }; + } + InflaterState::DecodingUncompressed => { + // Directly copy bytes from input to output. + let bytes_copied = self.output.copy_from(input, self.block_length); + self.block_length -= bytes_copied; + + if self.block_length == 0 { + // Done with this block, need to re-init bit buffer for next block + self.state = InflaterState::ReadingBFinal; + *end_of_block = true; + self.update_checkpoint_after_write_or_eob(input, true); + return Ok(()); + } + + self.update_checkpoint_after_write_or_eob(input, false); + + // We can fail to copy all bytes for two reasons: + // Running out of Input + // running out of free space in output window + if self.output.free_bytes() == 0 { + return Ok(()); + } + + return Err(InternalErr::DataNeeded); + } + _ => { + panic!("UnknownState"); + } + } + } + } + + fn decode_block( + &mut self, + input: &mut InputBuffer<'_>, + end_of_block_code_seen: &mut bool, + ) -> Result<(), InternalErr> { + *end_of_block_code_seen = false; + + if self.state == InflaterState::DecodeTop { + // Tight inner loop for decoding and processing deflate symbols when we know + // that there is both enough input available and also sufficient output space. + // State machine variables are not modified and self.state stays as DecodeTop. + match self.decode_block_fast_inner_loop(input) { + Ok((_, true)) => { + // End of block reached + *end_of_block_code_seen = true; + self.state = InflaterState::ReadingBFinal; + self.update_checkpoint_after_write_or_eob(input, true); + return Ok(()); + } + Ok((0, false)) => { + // No fast progress, fall through to slower but comprehensive + // state machine implementation which can load partial input. + } + Ok((_, false)) => { + // Some fast progress was made. Return so that output can be + // consumed by the caller and/or more input can be provided. + self.update_checkpoint_after_write_or_eob(input, false); + return Ok(()); + } + Err(InternalErr::DataError) => { + return Err(InternalErr::DataError); + } + Err(InternalErr::DataNeeded) => { + unreachable!("fast inner loop never returns DataNeeded") + } + } + } + + // State machine path + let mut free_bytes = self.output.free_bytes(); + while free_bytes >= TABLE_LOOKUP_LENGTH_MAX { + // With Deflate64 we can have up to a 64kb length, so we ensure at least that much space is available + // in the OutputWindow to avoid overwriting previous unflushed output data. + + let mut symbol; + match self.state { + InflaterState::DecodeTop => { + // decode an element from the literal tree + + // TODO: optimize this!!! + symbol = self.literal_length_tree.get_next_symbol(input)?; + + #[allow(clippy::comparison_chain)] + if symbol < 256 { + // literal + self.output.write(symbol as u8); + free_bytes -= 1; + self.update_checkpoint_after_write_or_eob(input, false); + } else if symbol == 256 { + // end of block + *end_of_block_code_seen = true; + // Reset state + self.state = InflaterState::ReadingBFinal; + self.update_checkpoint_after_write_or_eob(input, true); + return Ok(()); + } else { + // length/distance pair + symbol -= 257; // length code started at 257 + if symbol < 8 { + symbol += 3; // match length = 3,4,5,6,7,8,9,10 + self.extra_bits = 0; + } else if !self.deflate64 && symbol == 28 { + // extra bits for code 285 is 0 + symbol = 258; // code 285 means length 258 + self.extra_bits = 0; + } else { + if symbol as usize >= EXTRA_LENGTH_BITS.len() { + return Err(InternalErr::DataError); // GenericInvalidData + } + self.extra_bits = EXTRA_LENGTH_BITS[symbol as usize] as i32; + assert_ne!(self.extra_bits, 0, "We handle other cases separately!"); + } + self.length = symbol as usize; + + self.state = InflaterState::HaveInitialLength; + continue; //goto case InflaterState::HaveInitialLength; + } + } + InflaterState::HaveInitialLength => { + if self.extra_bits > 0 { + self.state = InflaterState::HaveInitialLength; + let bits = input.get_bits(self.extra_bits)?; + + if self.length >= LENGTH_BASE.len() { + return Err(InternalErr::DataError); // GenericInvalidData + } + self.length = LENGTH_BASE[self.length] as usize + bits as usize; + } + self.state = InflaterState::HaveFullLength; + continue; // goto case InflaterState::HaveFullLength; + } + InflaterState::HaveFullLength => { + if self.block_type == BlockType::Dynamic { + let bits = self.distance_tree.get_next_symbol(input)?; + self.distance_code = bits; + } else { + // get distance code directly for static block + let bits = input.get_bits(5)?; + self.distance_code = STATIC_DISTANCE_TREE_TABLE[bits as usize] as u16; + } + + self.state = InflaterState::HaveDistCode; + continue; //goto case InflaterState.HaveDistCode; + } + + InflaterState::HaveDistCode => { + // To avoid a table lookup we note that for distanceCode > 3, + // extra_bits = (distanceCode-2) >> 1 + let offset: usize; + if self.distance_code > 3 { + self.extra_bits = ((self.distance_code - 2) >> 1) as i32; + let bits = input.get_bits(self.extra_bits)?; + offset = DISTANCE_BASE_POSITION[self.distance_code as usize] as usize + + bits as usize; + } else { + offset = (self.distance_code + 1) as usize; + } + + if self.length > TABLE_LOOKUP_LENGTH_MAX || offset > TABLE_LOOKUP_DISTANCE_MAX { + return Err(InternalErr::DataError); + } + + self.output.write_length_distance(self.length, offset); + free_bytes -= self.length; + self.state = InflaterState::DecodeTop; + self.update_checkpoint_after_write_or_eob(input, false); + } + + _ => { + //Debug.Fail("check why we are here!"); + panic!("UnknownState"); + } + } + } + + Ok(()) + } + + /// Fast inner loop for decoding literals and length/distance pairs. Breaks out as soon as + /// maximum possible output cannot fit, or maximum possible input required is not available. + /// Returns (bytes_written, end_of_block) on success or InternalErr on failure. + fn decode_block_fast_inner_loop( + &mut self, + input: &mut InputBuffer<'_>, + ) -> Result<(usize, bool), InternalErr> { + let initial_free = self.output.free_bytes(); + + loop { + // Exit fast path if low on output space or input bits. + // Maximum input consumed per iteration is 64 bits, or 8 bytes: + // 16 bits for initial symbol value >= 257, indicating match length + // 16 bits for match length "extra bits" + // 16 bits for distance symbol + // 16 bits for distance "extra bits" + if self.output.free_bytes() < TABLE_LOOKUP_LENGTH_MAX || input.available_bytes() < 8 { + return Ok((initial_free - self.output.free_bytes(), false)); + } + + let symbol = self + .literal_length_tree + .get_next_symbol_assume_input(input)?; + match symbol { + 0..=255 => { + // Literal byte + self.output.write(symbol as u8); + } + 256 => { + // End of block + return Ok((initial_free - self.output.free_bytes(), true)); + } + 257..=285 => { + // Length/distance pair + let length_index = (symbol - 257) as usize; + let length = if length_index < 8 { + length_index + 3 + } else { + let extra_bits = EXTRA_LENGTH_BITS[length_index] as i32; + let bits = input.get_bits_assume_input(extra_bits); + LENGTH_BASE[length_index] as usize + bits as usize + }; + + let distance_code = if self.block_type == BlockType::Dynamic { + self.distance_tree.get_next_symbol_assume_input(input)? as usize + } else { + STATIC_DISTANCE_TREE_TABLE[input.get_bits_assume_input(5) as usize] as usize + }; + + let offset = if distance_code <= 3 { + distance_code + 1 + } else { + let extra_bits = ((distance_code - 2) >> 1) as i32; + let bits = input.get_bits_assume_input(extra_bits); + *DISTANCE_BASE_POSITION + .get(distance_code) + .ok_or(InternalErr::DataError)? as usize + + bits as usize + }; + + if length > TABLE_LOOKUP_LENGTH_MAX || offset > TABLE_LOOKUP_DISTANCE_MAX { + return Err(InternalErr::DataError); + } + self.output.write_length_distance(length, offset); + } + _ => { + // Symbol out of range + return Err(InternalErr::DataError); + } + } + } + } + + // Format of the dynamic block header: + // 5 Bits: HLIT, # of Literal/Length codes - 257 (257 - 286) + // 5 Bits: HDIST, # of Distance codes - 1 (1 - 32) + // 4 Bits: HCLEN, # of Code Length codes - 4 (4 - 19) + // + // (HCLEN + 4) x 3 bits: code lengths for the code length + // alphabet given just above, in the order: 16, 17, 18, + // 0, 8, 7, 9, 6, 10, 5, 11, 4, 12, 3, 13, 2, 14, 1, 15 + // + // These code lengths are interpreted as 3-bit integers + // (0-7); as above, a code length of 0 means the + // corresponding symbol (literal/length or distance code + // length) is not used. + // + // HLIT + 257 code lengths for the literal/length alphabet, + // encoded using the code length Huffman code + // + // HDIST + 1 code lengths for the distance alphabet, + // encoded using the code length Huffman code + // + // The code length repeat codes can cross from HLIT + 257 to the + // HDIST + 1 code lengths. In other words, all code lengths form + // a single sequence of HLIT + HDIST + 258 values. + fn decode_dynamic_block_header( + &mut self, + input: &mut InputBuffer<'_>, + ) -> Result<(), InternalErr> { + 'switch: loop { + match self.state { + InflaterState::ReadingNumLitCodes => { + let bits = input.get_bits(5)?; + self.literal_length_code_count = bits as u32 + 257; + self.state = InflaterState::ReadingNumDistCodes; + continue 'switch; //goto case InflaterState::ReadingNumDistCodes; + } + InflaterState::ReadingNumDistCodes => { + let bits = input.get_bits(5)?; + self.distance_code_count = bits as u32 + 1; + self.state = InflaterState::ReadingNumCodeLengthCodes; + continue 'switch; // goto case InflaterState::ReadingNumCodeLengthCodes; + } + InflaterState::ReadingNumCodeLengthCodes => { + let bits = input.get_bits(4)?; + self.code_length_code_count = bits as u32 + 4; + self.loop_counter = 0; + self.state = InflaterState::ReadingCodeLengthCodes; + continue 'switch; // goto case InflaterState::ReadingCodeLengthCodes; + } + InflaterState::ReadingCodeLengthCodes => { + while self.loop_counter < self.code_length_code_count { + let bits = input.get_bits(3)?; + self.code_length_tree_code_length + [CODE_ORDER[self.loop_counter as usize] as usize] = bits as u8; + self.loop_counter += 1; + } + + for &code_oder in &CODE_ORDER[self.code_length_code_count as usize..] { + self.code_length_tree_code_length[code_oder as usize] = 0; + } + + // create huffman tree for code length + self.code_length_tree + .new_in_place(&self.code_length_tree_code_length)?; + self.code_array_size = + self.literal_length_code_count + self.distance_code_count; + self.loop_counter = 0; // reset loop count + + self.state = InflaterState::ReadingTreeCodesBefore; + continue 'switch; // goto case InflaterState::ReadingTreeCodesBefore; + } + InflaterState::ReadingTreeCodesBefore | InflaterState::ReadingTreeCodesAfter => { + while self.loop_counter < self.code_array_size { + if self.state == InflaterState::ReadingTreeCodesBefore { + self.length_code = self.code_length_tree.get_next_symbol(input)?; + } + + // The alphabet for code lengths is as follows: + // 0 - 15: Represent code lengths of 0 - 15 + // 16: Copy the previous code length 3 - 6 times. + // The next 2 bits indicate repeat length + // (0 = 3, ... , 3 = 6) + // Example: Codes 8, 16 (+2 bits 11), + // 16 (+2 bits 10) will expand to + // 12 code lengths of 8 (1 + 6 + 5) + // 17: Repeat a code length of 0 for 3 - 10 times. + // (3 bits of length) + // 18: Repeat a code length of 0 for 11 - 138 times + // (7 bits of length) + if self.length_code <= 15 { + self.code_list[self.loop_counter as usize] = self.length_code as u8; + self.loop_counter += 1; + } else { + let repeat_count: u32; + if self.length_code == 16 { + self.state = InflaterState::ReadingTreeCodesAfter; + + if self.loop_counter == 0 { + // can't have "prev code" on first code + return Err(InternalErr::DataError); + } + + let bits = input.get_bits(2)?; + + let previous_code = self.code_list[self.loop_counter as usize - 1]; + repeat_count = (bits + 3) as u32; + + if self.loop_counter + repeat_count > self.code_array_size { + //throw new InvalidDataException(); + return Err(InternalErr::DataError); + } + + for _ in 0..repeat_count { + self.code_list[self.loop_counter as usize] = previous_code; + self.loop_counter += 1; + } + } else if self.length_code == 17 { + self.state = InflaterState::ReadingTreeCodesAfter; + let bits = input.get_bits(3)?; + + repeat_count = (bits + 3) as u32; + + if self.loop_counter + repeat_count > self.code_array_size { + //throw new InvalidDataException(); + return Err(InternalErr::DataError); + } + + for _ in 0..repeat_count { + self.code_list[self.loop_counter as usize] = 0; + self.loop_counter += 1; + } + } else { + // code == 18 + self.state = InflaterState::ReadingTreeCodesAfter; + let bits = input.get_bits(7)?; + + repeat_count = (bits + 11) as u32; + + if self.loop_counter + repeat_count > self.code_array_size { + //throw new InvalidDataException(); + return Err(InternalErr::DataError); + } + + for _ in 0..repeat_count { + self.code_list[self.loop_counter as usize] = 0; + self.loop_counter += 1; + } + } + } + self.state = InflaterState::ReadingTreeCodesBefore; // we want to read the next code. + } + break 'switch; + } + _ => { + panic!("InvalidDataException: UnknownState"); + } + } + } + + let mut literal_tree_code_length = [0u8; HuffmanTree::MAX_LITERAL_TREE_ELEMENTS]; + let mut distance_tree_code_length = [0u8; HuffmanTree::MAX_DIST_TREE_ELEMENTS]; + + // Create literal and distance tables + array_copy( + &self.code_list, + &mut literal_tree_code_length, + self.literal_length_code_count as usize, + ); + array_copy1( + &self.code_list, + self.literal_length_code_count as usize, + &mut distance_tree_code_length, + 0, + self.distance_code_count as usize, + ); + + // Make sure there is an end-of-block code, otherwise how could we ever end? + if literal_tree_code_length[HuffmanTree::END_OF_BLOCK_CODE] == 0 { + return Err(InternalErr::DataError); // InvalidDataException + } + + self.literal_length_tree + .new_in_place(&literal_tree_code_length)?; + self.distance_tree + .new_in_place(&distance_tree_code_length)?; + self.state = InflaterState::DecodeTop; + Ok(()) + } + + #[inline(always)] + #[allow(unused_variables)] + fn update_checkpoint_after_write_or_eob( + &mut self, + input: &InputBuffer<'_>, + end_of_block: bool, + ) { + #[cfg(feature = "checkpoint")] + checkpoint::update_checkpoint(self, input, end_of_block); + } +} + +#[cfg(feature = "checkpoint")] +#[path = "inflater_checkpoint.rs"] +pub mod checkpoint; diff --git a/patches/deflate64/src/input_buffer.rs b/patches/deflate64/src/input_buffer.rs new file mode 100644 index 00000000..a3edbc02 --- /dev/null +++ b/patches/deflate64/src/input_buffer.rs @@ -0,0 +1,203 @@ +use crate::InternalErr; +use std::cmp::min; + +#[derive(Copy, Clone, Debug)] +pub(crate) struct BitsBuffer { + pub(crate) bit_buffer: u32, + pub(crate) bits_in_buffer: i32, +} + +impl BitsBuffer { + pub(crate) fn new() -> BitsBuffer { + Self { + bit_buffer: 0, + bits_in_buffer: 0, + } + } + + #[allow(dead_code)] + pub(crate) fn from_bits(bits: u32, num_bits: i32) -> BitsBuffer { + debug_assert!((0..=32).contains(&num_bits)); + let mask = (!0u32).checked_shr(num_bits as u32).unwrap_or(0); + Self { + bit_buffer: bits & mask, + bits_in_buffer: num_bits, + } + } +} + +pub(crate) struct InputBuffer<'a> { + pub bits: BitsBuffer, + pub buffer: &'a [u8], + pub read_bytes: usize, +} + +impl<'a> InputBuffer<'a> { + pub fn new(bits: BitsBuffer, buffer: &'a [u8]) -> Self { + Self { + bits, + buffer, + read_bytes: 0, + } + } + + pub fn available_bits(&self) -> i32 { + self.bits.bits_in_buffer + } + + #[allow(dead_code)] + pub fn peek_available_bits(&self) -> u32 { + self.bits.bit_buffer + } + + pub fn available_bytes(&self) -> usize { + self.buffer.len() + (self.bits.bits_in_buffer / 8) as usize + } + + pub fn ensure_bits_available(&mut self, count: i32) -> bool { + debug_assert!(0 < count && count <= 16, "count is invalid."); + + // manual inlining to improve perf + if self.bits.bits_in_buffer < count { + if self.needs_input() { + return false; + } + + // insert a byte to bitbuffer + self.bits.bit_buffer |= (self.buffer[0] as u32) << self.bits.bits_in_buffer; + self.advance(1); + self.bits.bits_in_buffer += 8; + + if self.bits.bits_in_buffer < count { + if self.needs_input() { + return false; + } + // insert a byte to bitbuffer + self.bits.bit_buffer |= (self.buffer[0] as u32) << self.bits.bits_in_buffer; + self.advance(1); + self.bits.bits_in_buffer += 8; + } + } + + true + } + + pub fn try_load_16bits(&mut self) -> u32 { + if self.bits.bits_in_buffer < 8 { + if self.buffer.len() > 1 { + self.bits.bit_buffer |= (self.buffer[0] as u32) << self.bits.bits_in_buffer; + self.bits.bit_buffer |= (self.buffer[1] as u32) << (self.bits.bits_in_buffer + 8); + self.advance(2); + self.bits.bits_in_buffer += 16; + } else if !self.buffer.is_empty() { + self.bits.bit_buffer |= (self.buffer[0] as u32) << self.bits.bits_in_buffer; + self.advance(1); + self.bits.bits_in_buffer += 8; + } + } else if self.bits.bits_in_buffer < 16 && !self.buffer.is_empty() { + self.bits.bit_buffer |= (self.buffer[0] as u32) << self.bits.bits_in_buffer; + self.advance(1); + self.bits.bits_in_buffer += 8; + } + + self.bits.bit_buffer + } + + fn get_bit_mask(&self, count: i32) -> u32 { + (1 << count) - 1 + } + + pub fn get_bits(&mut self, count: i32) -> Result { + debug_assert!(0 < count && count <= 16, "count is invalid."); + + if !self.ensure_bits_available(count) { + return Err(InternalErr::DataNeeded); + } + + let result = (self.bits.bit_buffer & self.get_bit_mask(count)) as u16; + self.bits.bit_buffer >>= count; + self.bits.bits_in_buffer -= count; + Ok(result) + } + + // load_16bits_assume_input is an optimization of try_load_16bits when the caller knows + // that 16 bits are in the bit buffer or available as remaining input bytes. If this + // precondition is false, the call will assert in debug builds and fill with zeros in + // release builds (likely resulting in data error). + #[inline(always)] + pub fn load_16bits_assume_input(&mut self) -> u32 { + if self.bits.bits_in_buffer < 16 { + if let Some((head, tail)) = self.buffer.split_at_checked(2) { + let word = u16::from_le_bytes(head.try_into().unwrap()); + self.bits.bit_buffer |= (word as u32) << self.bits.bits_in_buffer; + self.buffer = tail; + self.read_bytes += 2; + } else { + debug_assert!(false, "Not enough input available"); + } + self.bits.bits_in_buffer += 16; + } + self.bits.bit_buffer + } + + // get_bits_assume_input is an optimization of get_bits using load_16bits_assume_input + // when the caller knows that 16 bits are in the bit buffer or available as remaining + // input bytes. If this precondition is false, the call will assert in debug builds and + // fill with zeros in release builds (likely resulting in data error). + #[inline(always)] + pub fn get_bits_assume_input(&mut self, count: i32) -> u32 { + debug_assert!(0 < count && count <= 16, "count is invalid."); + let result = self.load_16bits_assume_input() & self.get_bit_mask(count); + self.bits.bit_buffer >>= count; + self.bits.bits_in_buffer -= count; + result + } + + pub fn copy_to(&mut self, mut output: &mut [u8]) -> usize { + debug_assert!(self.bits.bits_in_buffer % 8 == 0); + + // Copy the bytes in bitBuffer first. + let mut bytes_from_bit_buffer = 0; + while self.bits.bits_in_buffer > 0 && !output.is_empty() { + output[0] = self.bits.bit_buffer as u8; + output = &mut output[1..]; + self.bits.bit_buffer >>= 8; + self.bits.bits_in_buffer -= 8; + bytes_from_bit_buffer += 1; + } + + if output.is_empty() { + return bytes_from_bit_buffer; + } + + let length = min(output.len(), self.buffer.len()); + output[..length].copy_from_slice(&self.buffer[..length]); + self.advance(length); + bytes_from_bit_buffer + length + } + + pub fn needs_input(&self) -> bool { + self.buffer.is_empty() + } + + /// Skip n bits in the buffer. + pub fn skip_bits(&mut self, n: i32) { + debug_assert!( + self.bits.bits_in_buffer >= n, + "No enough bits in the buffer, Did you call ensure_bits_available?" + ); + self.bits.bit_buffer >>= n; + self.bits.bits_in_buffer -= n; + } + + /// Skips to the next byte boundary. + pub fn skip_to_byte_boundary(&mut self) { + self.bits.bit_buffer >>= self.bits.bits_in_buffer % 8; + self.bits.bits_in_buffer -= self.bits.bits_in_buffer % 8; + } + + fn advance(&mut self, buf: usize) { + self.buffer = &self.buffer[buf..]; + self.read_bytes += buf; + } +} diff --git a/patches/deflate64/src/lib.rs b/patches/deflate64/src/lib.rs new file mode 100644 index 00000000..bb4d3fc4 --- /dev/null +++ b/patches/deflate64/src/lib.rs @@ -0,0 +1,135 @@ +//! Deflate64 implementation based on [.NET System.IO.Compression][dotnet]. +//! +//! This is made to unzip zip file with deflate64 made with windows 11. +//! +//! [dotnet]: https://github.com/dotnet/runtime/tree/e5efd8010e19593298dc2c3ee15106d5aec5a924/src/libraries/System.IO.Compression/src/System/IO/Compression/DeflateManaged + +#![forbid(unsafe_code)] +#![deny(rust_2018_idioms, nonstandard_style, future_incompatible)] +#![cfg_attr(docsrs, feature(doc_cfg))] + +mod buffer; +mod huffman_tree; +mod inflater_managed; +mod input_buffer; +mod output_window; +mod stream; + +#[cfg(feature = "checkpoint")] +pub use inflater_managed::checkpoint; +pub use inflater_managed::InflaterManaged; +pub use stream::Deflate64Decoder; + +#[derive(Debug, Copy, Clone, Eq, PartialEq)] +enum BlockType { + Uncompressed = 0, + Static = 1, + Dynamic = 2, +} + +impl BlockType { + pub fn from_int(int: u16) -> Option { + match int { + 0 => Some(Self::Uncompressed), + 1 => Some(Self::Static), + 2 => Some(Self::Dynamic), + _ => None, + } + } +} + +#[derive(Debug, Copy, Clone, Eq, PartialEq, Ord, PartialOrd)] +enum InflaterState { + //ReadingHeader = 0, // Only applies to GZIP + ReadingBFinal = 2, + // About to read bfinal bit + ReadingBType = 3, // About to read blockType bits + + ReadingNumLitCodes = 4, + // About to read # literal codes + ReadingNumDistCodes = 5, + // About to read # dist codes + ReadingNumCodeLengthCodes = 6, + // About to read # code length codes + ReadingCodeLengthCodes = 7, + // In the middle of reading the code length codes + ReadingTreeCodesBefore = 8, + // In the middle of reading tree codes (loop top) + ReadingTreeCodesAfter = 9, // In the middle of reading tree codes (extension; code > 15) + + DecodeTop = 10, + // About to decode a literal (char/match) in a compressed block + HaveInitialLength = 11, + // Decoding a match, have the literal code (base length) + HaveFullLength = 12, + // Ditto, now have the full match length (incl. extra length bits) + HaveDistCode = 13, // Ditto, now have the distance code also, need extra dist bits + + /* uncompressed blocks */ + UncompressedAligning = 15, + UncompressedByte1 = 16, + UncompressedByte2 = 17, + UncompressedByte3 = 18, + UncompressedByte4 = 19, + DecodingUncompressed = 20, + + // These three apply only to GZIP + //StartReadingFooter = 21, + // (Initialisation for reading footer) + //ReadingFooter = 22, + //VerifyingFooter = 23, + Done = 24, // Finished + + DataErrored = 100, +} + +impl std::ops::Sub for InflaterState { + type Output = u8; + + fn sub(self, rhs: Self) -> Self::Output { + self as u8 - rhs as u8 + } +} + +fn array_copy(source: &[T], dst: &mut [T], length: usize) { + dst[..length].copy_from_slice(&source[..length]); +} + +fn array_copy1( + source: &[T], + source_index: usize, + dst: &mut [T], + dst_index: usize, + length: usize, +) { + dst[dst_index..][..length].copy_from_slice(&source[source_index..][..length]); +} + +/// A structure containing result of streaming inflate. +#[derive(Debug)] +pub struct InflateResult { + /// The number of bytes consumed from the input slice. + pub bytes_consumed: usize, + /// The number of bytes written to the output slice. + pub bytes_written: usize, + /// true if there is error in input buffer + pub data_error: bool, +} + +impl InflateResult { + /// Creates `InflateResult` with zero bytes consumed and written, and no error. + #[allow(clippy::new_without_default)] + pub fn new() -> Self { + Self { + bytes_consumed: 0, + bytes_written: 0, + data_error: false, + } + } +} + +#[derive(Debug)] +enum InternalErr { + DataNeeded, + DataError, +} diff --git a/patches/deflate64/src/output_window.rs b/patches/deflate64/src/output_window.rs new file mode 100644 index 00000000..ebb03a7e --- /dev/null +++ b/patches/deflate64/src/output_window.rs @@ -0,0 +1,176 @@ +use crate::{buffer::Buffer, input_buffer::InputBuffer}; +use std::cmp::min; + +// With Deflate64 we can have up to a 65536 length as well as up to a 65538 distance. We need a power-of-two +// window size that goes back at least 65538 bytes, and we can only write into it when there are at least +// 65536 "free" bytes available for the maximum possible write length. However, it is OK if the free bytes +// overlap the history window; we process length-distance match copies in the forward direction. It is fine +// to wrap around and overwrite bytes that we have already copied forward. +pub(crate) const WINDOW_SIZE: usize = 131072; +const WINDOW_MASK: usize = 131071; + +/// +/// This class maintains a window for decompressed output. +/// We need to keep this because the decompressed information can be +/// a literal or a length/distance pair. For length/distance pair, +/// we need to look back in the output window and copy bytes from there. +/// We use a byte array of WINDOW_SIZE circularly. +/// +#[derive(Debug)] +pub(crate) struct OutputWindow { + window: [u8; WINDOW_SIZE], + end: usize, + bytes_used: usize, +} + +impl OutputWindow { + pub fn new() -> Self { + Self { + window: [0; WINDOW_SIZE], + end: 0, + bytes_used: 0, + } + } + + pub(crate) fn clear_bytes_used(&mut self) { + self.bytes_used = 0; + } + + /// Add a byte to output window. + #[inline(always)] + pub fn write(&mut self, b: u8) { + debug_assert!( + self.bytes_used < WINDOW_SIZE, + "Can't add byte when window is full!" + ); + self.window[self.end] = b; + self.end += 1; + self.end &= WINDOW_MASK; + self.bytes_used += 1; + } + + #[inline(always)] + pub fn write_length_distance(&mut self, length: usize, distance: usize) { + debug_assert!((self.bytes_used + length) <= WINDOW_SIZE, "No Enough space"); + + // move backwards distance bytes in the output stream, + // and copy length bytes from this position to the output stream. + + // This function *could* have lots of special-case optimizations for long + // non-overlapping copies, repeated bytes / patterns for long fills with + // short distances, separate paths for wrapping/non-wrapping writes, etc. + // but simpler ends up faster due to inlining and avoiding misprediction. + self.bytes_used += length; + let mut from = self.end.wrapping_sub(distance) & WINDOW_MASK; + let mut to = self.end; + + for _ in 0..length { + self.window[to] = self.window[from]; + to = (to + 1) & WINDOW_MASK; + from = (from + 1) & WINDOW_MASK; + } + + self.end = to; + } + + /// + /// Copy up to length of bytes from input directly. + /// This is used for uncompressed block. + /// + pub fn copy_from(&mut self, input: &mut InputBuffer<'_>, mut length: usize) -> usize { + length = min( + min(length, WINDOW_SIZE - self.bytes_used), + input.available_bytes(), + ); + let mut copied: usize; + + // We might need wrap around to copy all bytes. + let tail_len = WINDOW_SIZE - self.end; + if length > tail_len { + // copy the first part + copied = input.copy_to(&mut self.window[self.end..][..tail_len]); + if copied == tail_len { + // only try to copy the second part if we have enough bytes in input + copied += input.copy_to(&mut self.window[..length - tail_len]); + } + } else { + // only one copy is needed if there is no wrap around. + copied = input.copy_to(&mut self.window[self.end..][..length]); + } + + self.end = (self.end + copied) & WINDOW_MASK; + self.bytes_used += copied; + copied + } + + /// Free space in output window. + pub fn free_bytes(&self) -> usize { + WINDOW_SIZE - self.bytes_used + } + + /// Bytes not consumed in output window. + pub fn available_bytes(&self) -> usize { + self.bytes_used + } + + /// Copy the decompressed bytes to output buffer. + pub fn copy_to(&mut self, output: Buffer<'_>) -> usize { + let (copy_end, mut output) = if output.len() > self.bytes_used { + // we can copy all the decompressed bytes out + (self.end, output.index_mut(..self.bytes_used)) + } else { + // copy length of bytes + ( + (self + .end + .overflowing_sub(self.bytes_used) + .0 + .overflowing_add(output.len()) + .0) + & WINDOW_MASK, + output, + ) + }; + + let copied = output.len(); + + let mut output = if output.len() > copy_end { + let tail_len = output.len() - copy_end; + // this means we need to copy two parts separately + // copy the tail_len bytes from the end of the output window + output + .reborrow() + .index_mut(..tail_len) + .copy_from_slice(&self.window[WINDOW_SIZE - tail_len..][..tail_len]); + output.index_mut(tail_len..).index_mut(..copy_end) + } else { + output + }; + output.copy_from_slice(&self.window[copy_end - output.len()..][..output.len()]); + self.bytes_used -= copied; + //debug_assert!(self.bytes_used >= 0, "check this function and find why we copied more bytes than we have"); + copied + } + + #[cfg(feature = "checkpoint")] + pub(crate) fn get_checkpoint_data(&self, total_output_written: u64) -> (&[u8], &[u8]) { + use crate::inflater_managed::TABLE_LOOKUP_DISTANCE_MAX; + let history_needed = min(TABLE_LOOKUP_DISTANCE_MAX, total_output_written as usize); + let data_len = history_needed.max(self.bytes_used); + let start = (self.end + WINDOW_SIZE - data_len) & WINDOW_MASK; + if data_len <= WINDOW_SIZE - start { + // one contiguous range + (&self.window[start..start + data_len], &[]) + } else { + // wrap around, two ranges + (&self.window[start..], &self.window[..self.end]) + } + } + + #[cfg(feature = "checkpoint")] + pub(crate) fn restore_from_checkpoint(&mut self, data: &[u8], bytes_used: usize) { + self.window[..data.len()].copy_from_slice(data); + self.end = data.len(); + self.bytes_used = bytes_used; + } +} diff --git a/patches/deflate64/src/stream.rs b/patches/deflate64/src/stream.rs new file mode 100644 index 00000000..dcb625bb --- /dev/null +++ b/patches/deflate64/src/stream.rs @@ -0,0 +1,77 @@ +// TODO: move this module to deflate64 crate + +use crate::InflaterManaged; +use std::io::{self, BufRead, BufReader, Read}; + +/// The reader the decompresses deflate64 from another BufRead. +pub struct Deflate64Decoder { + inner: R, + inflater: Box, +} + +impl Deflate64Decoder> { + /// Creates Deflate64Decoder with Read + pub fn new(inner: R) -> Self { + Self::with_buffer(BufReader::new(inner)) + } +} + +impl Deflate64Decoder { + /// Creates Deflate64Decoder with BufRead + pub fn with_buffer(inner: R) -> Self { + Self { + inner, + inflater: Box::new(InflaterManaged::new()), + } + } +} + +impl Deflate64Decoder { + /// Returns inner BufRead instance + pub fn into_inner(self) -> R { + self.inner + } + + /// Returns reference to innner BufRead instance + pub fn get_ref(&self) -> &R { + &self.inner + } + + /// Returns mutable reference to innner BufRead instance + pub fn get_mut(&mut self) -> &mut R { + &mut self.inner + } +} + +impl Read for Deflate64Decoder { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + if buf.is_empty() { + // we received empty buffer, so it won't be possible to write anything + return Ok(0); + } + + loop { + let input = self.inner.fill_buf()?; + let eof = input.is_empty(); + + let result = self.inflater.inflate(input, buf); + + self.inner.consume(result.bytes_consumed); + + if result.data_error { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "invalid deflate64", + )); + } + + if result.bytes_written == 0 && !eof && !self.inflater.finished() { + // if we haven't ready any data and we haven't hit EOF yet, + // ask again. We must not return 0 in such case + continue; + } + + return Ok(result.bytes_written); + } + } +} diff --git a/src/bin/rofd/reader.rs b/src/bin/rofd/reader.rs index 8167d1fe..5840250f 100644 --- a/src/bin/rofd/reader.rs +++ b/src/bin/rofd/reader.rs @@ -77,7 +77,7 @@ pub fn match_snippets(text: &str, query: &str, limit: usize) -> Vec { for (i, ch) in chars.iter().enumerate() { for lower in ch.to_lowercase() { folded.push(lower); - positions.extend(std::iter::repeat_n(i, lower.len_utf8())); + positions.extend(std::iter::repeat(i).take(lower.len_utf8())); } } let query = query.to_lowercase();