diff options
author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
---|---|---|
committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-04-28 14:29:10 +0000 |
commit | 2aa4a82499d4becd2284cdb482213d541b8804dd (patch) | |
tree | b80bf8bf13c3766139fbacc530efd0dd9d54394c /third_party/rust/flate2 | |
parent | Initial commit. (diff) | |
download | firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.tar.xz firefox-2aa4a82499d4becd2284cdb482213d541b8804dd.zip |
Adding upstream version 86.0.1.upstream/86.0.1upstream
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'third_party/rust/flate2')
60 files changed, 8741 insertions, 0 deletions
diff --git a/third_party/rust/flate2/.cargo-checksum.json b/third_party/rust/flate2/.cargo-checksum.json new file mode 100644 index 0000000000..99cb4af7f0 --- /dev/null +++ b/third_party/rust/flate2/.cargo-checksum.json @@ -0,0 +1 @@ +{"files":{"Cargo.lock":"8057fb9114d5690ed7fd68e78a46c4663d5bd8c5df45f1852f3557cd887e82cc","Cargo.toml":"5d0e49885d316107f8ad83d091d5a4c3cbd80de64ccede374dd14ffb4fda95d0","LICENSE-APACHE":"a60eea817514531668d7e00765731449fe14d059d3249e0bc93b36de45f759f2","LICENSE-MIT":"378f5840b258e2779c39418f3f2d7b2ba96f1c7917dd6be0713f88305dbda397","README.md":"fbad05179827ff52e95df4015cae8ab8bb57df4d74bf767cb2fd710cd6528439","examples/compress_file.rs":"65cac39f50420861cb86120b0e7edce4704eb5afd1e27f790a66b8138176fc4d","examples/deflatedecoder-bufread.rs":"59aaa1dcf999da7909ea0a01bef757b80e366f8adf60984cc2e2c70acc774e1a","examples/deflatedecoder-read.rs":"7784b03156600d9a0efadb830920439a8a8f7fe70250090ee8c637033c1338eb","examples/deflatedecoder-write.rs":"6e6952cef15adbf465d4985802d315a3b51f3137be36e62bbcaed0bb27a8d5a6","examples/deflateencoder-bufread.rs":"0cefcb1edef04834e684c9c3d4048a32c7831617e51a102a7bd6a7351d4799be","examples/deflateencoder-read.rs":"56c65a0a1c951d0cf8281d0689bc520d7a9562e3e90fd35c333e4d832971e61d","examples/deflateencoder-write.rs":"79203d1db1128df2a3fa5cbd61d60776e02959c4da67a473f3b96dfb5f01f964","examples/gzbuilder.rs":"67ec1288b5ccb800a5f20148be61e47c7ebacdc1b96d404e2810f397314b39b3","examples/gzdecoder-bufread.rs":"e19ebd1085eeb9cf49b081953184207b949464ece5add8bb75700fdb514f2cad","examples/gzdecoder-read.rs":"0a191c6215e8eed01b9cfaa10804869693e2d4fbff29e8d5469fbabc65ded610","examples/gzdecoder-write.rs":"a8843c75fa725de84b8eb75b08525e61586f60f9f8bfd874fbfd44039d119c5f","examples/gzencoder-bufread.rs":"aaad311ad799c2e739b58fc4e570225d8f52d3bc407f9c71c28023b321e02100","examples/gzencoder-read.rs":"d28ec1d0ab3d4dabb6f0d98917efd4a8b72ff0a70b2b8596009c2d6e86328226","examples/gzencoder-write.rs":"2eea8a8c8bf22f060d8749d786835ce9454abe2cfd24bf0f7a23b5a8d1030121","examples/gzmultidecoder-bufread.rs":"1e4b31da4f1352e748b4516ba60dcd52e5d5e62733fa1c8e7da7bee4efcce1c4","examples/gzmultidecoder-read.rs":"3976274ac8b8190b3f7ed9fd855a193c38458bd40a2dba6a7de21986f78682f6","examples/hello_world.txt":"d2a84f4b8b650937ec8f73cd8be2c74add5a911ba64df27458ed8229da804a26","examples/zlibdecoder-bufread.rs":"e57a597030e6bd87e79329d87b3743b7fb0ff4c217846b0ba2c2d32264208ff1","examples/zlibdecoder-read.rs":"83b3abd1c12586f820d55e4d86c328fdccbdec963e90c44a44cd7d5f1caa4c8f","examples/zlibdecoder-write.rs":"e7cbec46414eb5d67fee95bf1e4a4e6a1ba2a41f4ce9542dd4481035d3f8070d","examples/zlibencoder-bufread.rs":"13d79baeffb6080b4b4f837e23092b52b92b85466ed9e42b0a33a864232e693f","examples/zlibencoder-read.rs":"120496c8d0854d13914fd5d23cc78994b6c9450ca97dead668eb4b616bbfcdda","examples/zlibencoder-write.rs":"47c25fb6711e22f95bdc89309c6e367d1060a0120d4259011e97908563e2b8bc","src/bufreader.rs":"7fc0b74ca751e9e11689d3e57abc3aadbda92f25f2a9a80fca009f43a112642d","src/crc.rs":"fa8b95d789848ccd835af17f0d733ccebdbf260feadd69e0f0016f7d94644582","src/deflate/bufread.rs":"bb4e02f5c292105def3cee998d4f5163cafcf3a9eede0b1ba2071f8366825c56","src/deflate/mod.rs":"c76346b014be16947e836e752a3e88095471f7656b894b1b2789ce5fbd130794","src/deflate/read.rs":"1824254346d7b95f56611be38b6baca154fa45b37866cbc93195a52522925e41","src/deflate/write.rs":"08dafb0ccc5e6a92d667470d2b8e9aa8e828001375e144522813b052d4a8463f","src/ffi/c.rs":"c75b7a1e878fa0e1f9e5e8c9f7fd9c150960c7863a23995cdc0e0edf39594360","src/ffi/mod.rs":"c5106dc28233c97610c321456a69b3a7ffb4c03442112cccb4174eac871e2fe3","src/ffi/rust.rs":"909dec8e979e5b076ef3fd118e3f8aedea68c8011000869e26497d2faedf2348","src/gz/bufread.rs":"1eb3ddccceda75a1574663f5583bf4ab809ee5f0ff690a29ea0e08f857eb933e","src/gz/mod.rs":"798203ecb5b05ebbdd8d2e5705794300a84ca07f2f78aa52549be0df649daf14","src/gz/read.rs":"51404bb8477aad79816b716e1387abc18155bd11225597a374d115b8b6ffb41f","src/gz/write.rs":"506aee09388008f60a2ad86e1766e44ae5546fa8bce52ceccc6df95c1350e3db","src/lib.rs":"e6b230a5dcd76d143986fcde7207a33a0c194bd5b54403984b6acd68844b2bb8","src/mem.rs":"6a85ee716e186d66826d87eaec9b05265ab2deb10d802cd177d21c332551ec69","src/zio.rs":"a7d50602b06c156bd3f520c80b9e1c273234f18a0213e2789896dea4e8e85588","src/zlib/bufread.rs":"bb9d158f2005acb468bf81f66fd6dcc0f2b3e8fdf9ceb96626315e62029f8fd4","src/zlib/mod.rs":"facb051b288a065cdcce92a0c42b546c53264f6facd1bbd3cbf14a79c93c6dcd","src/zlib/read.rs":"660fa0f54417193a0a5d318aa8e3ab995e3228ac7d37e088efd5198394c27fc2","src/zlib/write.rs":"a3ac0303870ddf3231a7d928ae83ed494f0f48311f2295913632db0c6c490ef4","tests/async-reader.rs":"7a1d4fb1324a075ed2dee0d10f932d1ac8ddbc6f80befea95d65cda390491b1e","tests/corrupt-file.gz":"083dd284aa1621916a2d0f66ea048c8d3ba7a722b22d0d618722633f51e7d39c","tests/early-flush.rs":"5ed4b0f8e66cab9209e079d5636e5f1b780606cd553a182c809cf73a24e77e8e","tests/empty-read.rs":"45477d316f77a048d747e9d18292abfec0cac667768385c8a061e18fd1240238","tests/good-file.gz":"87296963e53024a74752179ce7e54087565d358a85d3e65c3b37ef36eaa3d4a6","tests/good-file.txt":"bc4e03658a441fe2ad2df7cd2197144b87e41696f01e327b380e869cd9b485a0","tests/gunzip.rs":"623a90c6688c850d3c46d23acabc0f88a17d3b86c90d269ecc0766dc7138943d","tests/multi.gz":"efa3341da052f95056314cc6920e02a3da15bdef30234b2552fb407812db5cc6","tests/multi.txt":"dbea9325179efe46ea2add94f7b6b745ca983fabb208dc6d34aa064623d7ee23","tests/tokio.rs":"6a5a0fa838b81ef747bb3eed8ea09111132da71f44de5dfba64d47f4ec5a4e8f","tests/zero-write.rs":"ff8d0349a540b54363c55807c5fd7fbbdc363d08a536d35a3a40f0ce92c16489"},"package":"7411863d55df97a419aa64cb4d2f167103ea9d767e2c54a1868b7ac3f6b47129"}
\ No newline at end of file diff --git a/third_party/rust/flate2/Cargo.lock b/third_party/rust/flate2/Cargo.lock new file mode 100644 index 0000000000..0b7eea527c --- /dev/null +++ b/third_party/rust/flate2/Cargo.lock @@ -0,0 +1,630 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +[[package]] +name = "adler" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e" + +[[package]] +name = "autocfg" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" + +[[package]] +name = "bitflags" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" + +[[package]] +name = "byteorder" +version = "1.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" + +[[package]] +name = "bytes" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "206fdffcfa2df7cbe15601ef46c813fce0965eb3286db6b56c583b814b51c81c" +dependencies = [ + "byteorder", + "iovec", +] + +[[package]] +name = "cc" +version = "1.0.61" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" + +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "cloudabi" +version = "0.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddfc5b9aa5d4507acaf872de71051dfd0e309860e88966e1051e462a077aac4f" +dependencies = [ + "bitflags", +] + +[[package]] +name = "cloudflare-zlib-sys" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e195cb274a0d6ee87e718838a09baecd7cbc9f6075dac256a84cb5842739c06" +dependencies = [ + "cc", +] + +[[package]] +name = "cmake" +version = "0.1.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0e56268c17a6248366d66d4a47a3381369d068cce8409bb1716ed77ea32163bb" +dependencies = [ + "cc", +] + +[[package]] +name = "crc32fast" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81156fece84ab6a9f2afdb109ce3ae577e42b1228441eded99bd77f627953b1a" +dependencies = [ + "cfg-if 1.0.0", +] + +[[package]] +name = "crossbeam-deque" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f02af974daeee82218205558e51ec8768b48cf524bd01d550abe5573a608285" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "crossbeam-utils", + "lazy_static", + "maybe-uninit", + "memoffset", + "scopeguard", +] + +[[package]] +name = "crossbeam-queue" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" +dependencies = [ + "cfg-if 0.1.10", + "crossbeam-utils", + "maybe-uninit", +] + +[[package]] +name = "crossbeam-utils" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" +dependencies = [ + "autocfg", + "cfg-if 0.1.10", + "lazy_static", +] + +[[package]] +name = "flate2" +version = "1.0.19" +dependencies = [ + "cfg-if 1.0.0", + "cloudflare-zlib-sys", + "crc32fast", + "futures", + "libc", + "libz-sys", + "miniz-sys", + "miniz_oxide", + "quickcheck", + "rand", + "tokio-io", + "tokio-tcp", + "tokio-threadpool", +] + +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + +[[package]] +name = "fuchsia-zircon" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" +dependencies = [ + "bitflags", + "fuchsia-zircon-sys", +] + +[[package]] +name = "fuchsia-zircon-sys" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" + +[[package]] +name = "futures" +version = "0.1.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c7e4c2612746b0df8fed4ce0c69156021b704c9aefa360311c04e6e9e002eed" + +[[package]] +name = "getrandom" +version = "0.1.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "wasi", +] + +[[package]] +name = "hermit-abi" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +dependencies = [ + "libc", +] + +[[package]] +name = "iovec" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" +dependencies = [ + "libc", +] + +[[package]] +name = "kernel32-sys" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] + +[[package]] +name = "lazy_static" +version = "1.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" + +[[package]] +name = "libc" +version = "0.2.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4d58d1b70b004888f764dfbf6a26a3b0342a1632d33968e4a179d8011c760614" + +[[package]] +name = "libz-sys" +version = "1.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "602113192b08db8f38796c4e85c39e960c145965140e918018bcde1952429655" +dependencies = [ + "cc", + "cmake", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "lock_api" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4da24a77a3d8a6d4862d95f72e6fdb9c09a643ecdb402d754004a557f2bec75" +dependencies = [ + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +dependencies = [ + "cfg-if 0.1.10", +] + +[[package]] +name = "maybe-uninit" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" + +[[package]] +name = "memoffset" +version = "0.5.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "043175f069eda7b85febe4a74abbaeff828d9f8b448515d3151a14a3542811aa" +dependencies = [ + "autocfg", +] + +[[package]] +name = "miniz-sys" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e9e3ae51cea1576ceba0dde3d484d30e6e5b86dee0b2d412fe3a16a15c98202" +dependencies = [ + "cc", + "libc", +] + +[[package]] +name = "miniz_oxide" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f2d26ec3309788e423cfbf68ad1800f061638098d76a83681af979dc4eda19d" +dependencies = [ + "adler", + "autocfg", +] + +[[package]] +name = "mio" +version = "0.6.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" +dependencies = [ + "cfg-if 0.1.10", + "fuchsia-zircon", + "fuchsia-zircon-sys", + "iovec", + "kernel32-sys", + "libc", + "log", + "miow", + "net2", + "slab", + "winapi 0.2.8", +] + +[[package]] +name = "miow" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" +dependencies = [ + "kernel32-sys", + "net2", + "winapi 0.2.8", + "ws2_32-sys", +] + +[[package]] +name = "net2" +version = "0.2.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" +dependencies = [ + "cfg-if 0.1.10", + "libc", + "winapi 0.3.9", +] + +[[package]] +name = "num_cpus" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "parking_lot" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f842b1982eb6c2fe34036a4fbfb06dd185a3f5c8edfaacdf7d1ea10b07de6252" +dependencies = [ + "lock_api", + "parking_lot_core", + "rustc_version", +] + +[[package]] +name = "parking_lot_core" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b876b1b9e7ac6e1a74a6da34d25c42e17e8862aa409cbbbdcfc8d86c6f3bc62b" +dependencies = [ + "cfg-if 0.1.10", + "cloudabi", + "libc", + "redox_syscall", + "rustc_version", + "smallvec", + "winapi 0.3.9", +] + +[[package]] +name = "pkg-config" +version = "0.3.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c" + +[[package]] +name = "ppv-lite86" +version = "0.2.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" + +[[package]] +name = "quickcheck" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44883e74aa97ad63db83c4bf8ca490f02b2fc02f92575e720c8551e843c945f" +dependencies = [ + "rand", + "rand_core", +] + +[[package]] +name = "rand" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" +dependencies = [ + "getrandom", + "libc", + "rand_chacha", + "rand_core", + "rand_hc", +] + +[[package]] +name = "rand_chacha" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" +dependencies = [ + "ppv-lite86", + "rand_core", +] + +[[package]] +name = "rand_core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" +dependencies = [ + "getrandom", +] + +[[package]] +name = "rand_hc" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" +dependencies = [ + "rand_core", +] + +[[package]] +name = "redox_syscall" +version = "0.1.57" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" + +[[package]] +name = "rustc_version" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" +dependencies = [ + "semver", +] + +[[package]] +name = "scopeguard" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" + +[[package]] +name = "semver" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403" +dependencies = [ + "semver-parser", +] + +[[package]] +name = "semver-parser" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3" + +[[package]] +name = "slab" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" + +[[package]] +name = "smallvec" +version = "0.6.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7b0758c52e15a8b5e3691eae6cc559f08eee9406e548a4477ba4e67770a82b6" +dependencies = [ + "maybe-uninit", +] + +[[package]] +name = "tokio-executor" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb2d1b8f4548dbf5e1f7818512e9c406860678f29c300cdf0ebac72d1a3a1671" +dependencies = [ + "crossbeam-utils", + "futures", +] + +[[package]] +name = "tokio-io" +version = "0.1.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fc868aae093479e3131e3d165c93b1c7474109d13c90ec0dda2a1bbfff0674" +dependencies = [ + "bytes", + "futures", + "log", +] + +[[package]] +name = "tokio-reactor" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09bc590ec4ba8ba87652da2068d150dcada2cfa2e07faae270a5e0409aa51351" +dependencies = [ + "crossbeam-utils", + "futures", + "lazy_static", + "log", + "mio", + "num_cpus", + "parking_lot", + "slab", + "tokio-executor", + "tokio-io", + "tokio-sync", +] + +[[package]] +name = "tokio-sync" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "edfe50152bc8164fcc456dab7891fa9bf8beaf01c5ee7e1dd43a397c3cf87dee" +dependencies = [ + "fnv", + "futures", +] + +[[package]] +name = "tokio-tcp" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98df18ed66e3b72e742f185882a9e201892407957e45fbff8da17ae7a7c51f72" +dependencies = [ + "bytes", + "futures", + "iovec", + "mio", + "tokio-io", + "tokio-reactor", +] + +[[package]] +name = "tokio-threadpool" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df720b6581784c118f0eb4310796b12b1d242a7eb95f716a8367855325c25f89" +dependencies = [ + "crossbeam-deque", + "crossbeam-queue", + "crossbeam-utils", + "futures", + "lazy_static", + "log", + "num_cpus", + "slab", + "tokio-executor", +] + +[[package]] +name = "vcpkg" +version = "0.2.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6454029bf181f092ad1b853286f23e2c507d8e8194d01d92da4a55c274a5508c" + +[[package]] +name = "wasi" +version = "0.9.0+wasi-snapshot-preview1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" + +[[package]] +name = "winapi" +version = "0.2.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" + +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-build" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + +[[package]] +name = "ws2_32-sys" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" +dependencies = [ + "winapi 0.2.8", + "winapi-build", +] diff --git a/third_party/rust/flate2/Cargo.toml b/third_party/rust/flate2/Cargo.toml new file mode 100644 index 0000000000..1b54c6f45e --- /dev/null +++ b/third_party/rust/flate2/Cargo.toml @@ -0,0 +1,89 @@ +# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO +# +# When uploading crates to the registry Cargo will automatically +# "normalize" Cargo.toml files for maximal compatibility +# with all versions of Cargo and also rewrite `path` dependencies +# to registry (e.g., crates.io) dependencies +# +# If you believe there's an error in this file please file an +# issue against the rust-lang/cargo repository. If you're +# editing this file be aware that the upstream Cargo.toml +# will likely look very different (and much more reasonable) + +[package] +edition = "2018" +name = "flate2" +version = "1.0.19" +authors = ["Alex Crichton <alex@alexcrichton.com>", "Josh Triplett <josh@joshtriplett.org>"] +description = "DEFLATE compression and decompression exposed as Read/BufRead/Write streams.\nSupports miniz_oxide, miniz.c, and multiple zlib implementations. Supports\nzlib, gzip, and raw deflate streams.\n" +homepage = "https://github.com/rust-lang/flate2-rs" +documentation = "https://docs.rs/flate2" +readme = "README.md" +keywords = ["gzip", "deflate", "zlib", "zlib-ng", "encoding"] +categories = ["compression", "api-bindings"] +license = "MIT/Apache-2.0" +repository = "https://github.com/rust-lang/flate2-rs" +[dependencies.cfg-if] +version = "1.0.0" + +[dependencies.cloudflare-zlib-sys] +version = "0.2.0" +optional = true + +[dependencies.crc32fast] +version = "1.2.0" + +[dependencies.futures] +version = "0.1.25" +optional = true + +[dependencies.libc] +version = "0.2.65" + +[dependencies.libz-sys] +version = "1.1.0" +optional = true +default-features = false + +[dependencies.miniz-sys] +version = "0.1.11" +optional = true + +[dependencies.miniz_oxide] +version = "0.4.0" +optional = true +default-features = false + +[dependencies.tokio-io] +version = "0.1.11" +optional = true +[dev-dependencies.futures] +version = "0.1" + +[dev-dependencies.quickcheck] +version = "0.9" +default-features = false + +[dev-dependencies.rand] +version = "0.7" + +[dev-dependencies.tokio-io] +version = "0.1.11" + +[dev-dependencies.tokio-tcp] +version = "0.1.3" + +[dev-dependencies.tokio-threadpool] +version = "0.1.10" + +[features] +any_zlib = [] +cloudflare_zlib = ["any_zlib", "cloudflare-zlib-sys"] +default = ["rust_backend"] +rust_backend = ["miniz_oxide"] +tokio = ["tokio-io", "futures"] +zlib = ["any_zlib", "libz-sys"] +zlib-ng-compat = ["zlib", "libz-sys/zlib-ng"] +[target."cfg(all(target_arch = \"wasm32\", not(target_os = \"emscripten\")))".dependencies.miniz_oxide] +version = "0.4.0" +default-features = false diff --git a/third_party/rust/flate2/LICENSE-APACHE b/third_party/rust/flate2/LICENSE-APACHE new file mode 100644 index 0000000000..16fe87b06e --- /dev/null +++ b/third_party/rust/flate2/LICENSE-APACHE @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + +2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + +3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + +4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + +5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + +6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + +8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + +Copyright [yyyy] [name of copyright owner] + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/third_party/rust/flate2/LICENSE-MIT b/third_party/rust/flate2/LICENSE-MIT new file mode 100644 index 0000000000..39e0ed6602 --- /dev/null +++ b/third_party/rust/flate2/LICENSE-MIT @@ -0,0 +1,25 @@ +Copyright (c) 2014 Alex Crichton + +Permission is hereby granted, free of charge, to any +person obtaining a copy of this software and associated +documentation files (the "Software"), to deal in the +Software without restriction, including without +limitation the rights to use, copy, modify, merge, +publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software +is furnished to do so, subject to the following +conditions: + +The above copyright notice and this permission notice +shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF +ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED +TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A +PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT +SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR +IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. diff --git a/third_party/rust/flate2/README.md b/third_party/rust/flate2/README.md new file mode 100644 index 0000000000..3c6ed76399 --- /dev/null +++ b/third_party/rust/flate2/README.md @@ -0,0 +1,105 @@ +# flate2 + +[![Crates.io](https://img.shields.io/crates/v/flate2.svg?maxAge=2592000)](https://crates.io/crates/flate2) +[![Documentation](https://docs.rs/flate2/badge.svg)](https://docs.rs/flate2) + +A streaming compression/decompression library DEFLATE-based streams in Rust. + +This crate by default uses the `miniz_oxide` crate, a port of `miniz.c` to pure +Rust. This crate also supports other [backends](#Backends), such as the widely +available zlib library or the high-performance zlib-ng library. + +Supported formats: + +* deflate +* zlib +* gzip + +```toml +# Cargo.toml +[dependencies] +flate2 = "1.0" +``` + +## Compression + +```rust +use std::io::prelude::*; +use flate2::Compression; +use flate2::write::ZlibEncoder; + +fn main() { + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"foo"); + e.write_all(b"bar"); + let compressed_bytes = e.finish(); +} +``` + +## Decompression + +```rust,no_run +use std::io::prelude::*; +use flate2::read::GzDecoder; + +fn main() { + let mut d = GzDecoder::new("...".as_bytes()); + let mut s = String::new(); + d.read_to_string(&mut s).unwrap(); + println!("{}", s); +} +``` + +## Backends + +The default `miniz_oxide` backend has the advantage of being pure Rust, but if +you're already using zlib with another C library, for example, you can use that +for Rust code as well: + +```toml +[dependencies] +flate2 = { version = "1.0.17", features = ["zlib"], default-features = false } +``` + +This supports either the high-performance zlib-ng backend (in zlib-compat mode) +or the use of a shared system zlib library. To explicitly opt into the fast +zlib-ng backend, use: + +```toml +[dependencies] +flate2 = { version = "1.0.17", features = ["zlib-ng-compat"], default-features = false } +``` + +Note that if any crate in your dependency graph explicitly requests stock zlib, +or uses libz-sys directly without `default-features = false`, you'll get stock +zlib rather than zlib-ng. See [the libz-sys +README](https://github.com/rust-lang/libz-sys/blob/main/README.md) for details. + +For compatibility with previous versions of `flate2`, the cloudflare optimized +version of zlib is available, via the `cloudflare_zlib` feature. It's not as +fast as zlib-ng, but it's faster than stock zlib. It requires a x86-64 CPU with +SSE 4.2 or ARM64 with NEON & CRC. It does not support 32-bit CPUs at all and is +incompatible with mingw. For more information check the [crate +documentation](https://crates.io/crates/cloudflare-zlib-sys). Note that +`cloudflare_zlib` will cause breakage if any other crate in your crate graph +uses another version of zlib/libz. + +For compatibility with previous versions of `flate2`, the C version of `miniz.c` +is still available, using the feature `miniz-sys`. + +# License + +This project is licensed under either of + + * Apache License, Version 2.0, ([LICENSE-APACHE](LICENSE-APACHE) or + http://www.apache.org/licenses/LICENSE-2.0) + * MIT license ([LICENSE-MIT](LICENSE-MIT) or + http://opensource.org/licenses/MIT) + +at your option. + +### Contribution + +Unless you explicitly state otherwise, any contribution intentionally submitted +for inclusion in this project by you, as defined in the Apache-2.0 license, +shall be dual licensed as above, without any additional terms or conditions. diff --git a/third_party/rust/flate2/examples/compress_file.rs b/third_party/rust/flate2/examples/compress_file.rs new file mode 100644 index 0000000000..39ed8eed4e --- /dev/null +++ b/third_party/rust/flate2/examples/compress_file.rs @@ -0,0 +1,28 @@ +extern crate flate2; + +use flate2::write::GzEncoder; +use flate2::Compression; +use std::env::args; +use std::fs::File; +use std::io::copy; +use std::io::BufReader; +use std::time::Instant; + +fn main() { + if args().len() != 3 { + eprintln!("Usage: ./compress_file `source` `target`"); + return; + } + let mut input = BufReader::new(File::open(args().nth(1).unwrap()).unwrap()); + let output = File::create(args().nth(2).unwrap()).unwrap(); + let mut encoder = GzEncoder::new(output, Compression::default()); + let start = Instant::now(); + copy(&mut input, &mut encoder).unwrap(); + let output = encoder.finish().unwrap(); + println!( + "Source len: {:?}", + input.get_ref().metadata().unwrap().len() + ); + println!("Target len: {:?}", output.metadata().unwrap().len()); + println!("Elapsed: {:?}", start.elapsed()); +} diff --git a/third_party/rust/flate2/examples/deflatedecoder-bufread.rs b/third_party/rust/flate2/examples/deflatedecoder-bufread.rs new file mode 100644 index 0000000000..7d1cb6fd90 --- /dev/null +++ b/third_party/rust/flate2/examples/deflatedecoder-bufread.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::bufread::DeflateDecoder; +use flate2::write::DeflateEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = DeflateEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Deflate Encoded vector of bytes and returns a string or error +// Here &[u8] implements Read +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut deflater = DeflateDecoder::new(&bytes[..]); + let mut s = String::new(); + deflater.read_to_string(&mut s)?; + Ok(s) +} diff --git a/third_party/rust/flate2/examples/deflatedecoder-read.rs b/third_party/rust/flate2/examples/deflatedecoder-read.rs new file mode 100644 index 0000000000..cc5d435be3 --- /dev/null +++ b/third_party/rust/flate2/examples/deflatedecoder-read.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::read::DeflateDecoder; +use flate2::write::DeflateEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = DeflateEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Deflate Encoded vector of bytes and returns a string or error +// Here &[u8] implements Read +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut deflater = DeflateDecoder::new(&bytes[..]); + let mut s = String::new(); + deflater.read_to_string(&mut s)?; + Ok(s) +} diff --git a/third_party/rust/flate2/examples/deflatedecoder-write.rs b/third_party/rust/flate2/examples/deflatedecoder-write.rs new file mode 100644 index 0000000000..276490bac2 --- /dev/null +++ b/third_party/rust/flate2/examples/deflatedecoder-write.rs @@ -0,0 +1,26 @@ +extern crate flate2; + +use flate2::write::DeflateDecoder; +use flate2::write::DeflateEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = DeflateEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Deflate Encoded vector of bytes and returns a string or error +// Here Vec<u8> implements Write +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut writer = Vec::new(); + let mut deflater = DeflateDecoder::new(writer); + deflater.write_all(&bytes[..])?; + writer = deflater.finish()?; + let return_string = String::from_utf8(writer).expect("String parsing error"); + Ok(return_string) +} diff --git a/third_party/rust/flate2/examples/deflateencoder-bufread.rs b/third_party/rust/flate2/examples/deflateencoder-bufread.rs new file mode 100644 index 0000000000..6240f39972 --- /dev/null +++ b/third_party/rust/flate2/examples/deflateencoder-bufread.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::bufread::DeflateEncoder; +use flate2::Compression; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::io::BufReader; + +// Open file and debug print the contents compressed with Deflate +fn main() { + println!("{:?}", open_hello_world().unwrap()); +} + +// Opens sample file, compresses the contents and returns a Vector or error +// File wrapped in a BufReader implements Bufread +fn open_hello_world() -> io::Result<Vec<u8>> { + let f = File::open("examples/hello_world.txt")?; + let b = BufReader::new(f); + let mut deflater = DeflateEncoder::new(b, Compression::fast()); + let mut buffer = Vec::new(); + deflater.read_to_end(&mut buffer)?; + Ok(buffer) +} diff --git a/third_party/rust/flate2/examples/deflateencoder-read.rs b/third_party/rust/flate2/examples/deflateencoder-read.rs new file mode 100644 index 0000000000..d22777f51f --- /dev/null +++ b/third_party/rust/flate2/examples/deflateencoder-read.rs @@ -0,0 +1,20 @@ +extern crate flate2; + +use flate2::read::DeflateEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Print the Deflate compressed representation of hello world +fn main() { + println!("{:?}", deflateencoder_read_hello_world().unwrap()); +} + +// Return a vector containing the Defalte compressed version of hello world +fn deflateencoder_read_hello_world() -> io::Result<Vec<u8>> { + let mut ret_vec = [0; 100]; + let c = b"hello world"; + let mut deflater = DeflateEncoder::new(&c[..], Compression::fast()); + let count = deflater.read(&mut ret_vec)?; + Ok(ret_vec[0..count].to_vec()) +} diff --git a/third_party/rust/flate2/examples/deflateencoder-write.rs b/third_party/rust/flate2/examples/deflateencoder-write.rs new file mode 100644 index 0000000000..243b9dfdd0 --- /dev/null +++ b/third_party/rust/flate2/examples/deflateencoder-write.rs @@ -0,0 +1,12 @@ +extern crate flate2; + +use flate2::write::DeflateEncoder; +use flate2::Compression; +use std::io::prelude::*; + +// Vec<u8> implements Write to print the compressed bytes of sample string +fn main() { + let mut e = DeflateEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + println!("{:?}", e.finish().unwrap()); +} diff --git a/third_party/rust/flate2/examples/gzbuilder.rs b/third_party/rust/flate2/examples/gzbuilder.rs new file mode 100644 index 0000000000..031683decf --- /dev/null +++ b/third_party/rust/flate2/examples/gzbuilder.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::Compression; +use flate2::GzBuilder; +use std::fs::File; +use std::io; +use std::io::prelude::*; + +// Open file and debug print the contents compressed with gzip +fn main() { + sample_builder().unwrap(); +} + +// GzBuilder opens a file and writes a sample string using Builder pattern +fn sample_builder() -> Result<(), io::Error> { + let f = File::create("examples/hello_world.gz")?; + let mut gz = GzBuilder::new() + .filename("hello_world.txt") + .comment("test file, please delete") + .write(f, Compression::default()); + gz.write_all(b"hello world")?; + gz.finish()?; + Ok(()) +} diff --git a/third_party/rust/flate2/examples/gzdecoder-bufread.rs b/third_party/rust/flate2/examples/gzdecoder-bufread.rs new file mode 100644 index 0000000000..0702c174cd --- /dev/null +++ b/third_party/rust/flate2/examples/gzdecoder-bufread.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::bufread::GzDecoder; +use flate2::write::GzEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Gz Encoded vector of bytes and returns a string or error +// Here &[u8] implements BufRead +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut gz = GzDecoder::new(&bytes[..]); + let mut s = String::new(); + gz.read_to_string(&mut s)?; + Ok(s) +} diff --git a/third_party/rust/flate2/examples/gzdecoder-read.rs b/third_party/rust/flate2/examples/gzdecoder-read.rs new file mode 100644 index 0000000000..d8dcba3671 --- /dev/null +++ b/third_party/rust/flate2/examples/gzdecoder-read.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Gz Encoded vector of bytes and returns a string or error +// Here &[u8] implements Read +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut gz = GzDecoder::new(&bytes[..]); + let mut s = String::new(); + gz.read_to_string(&mut s)?; + Ok(s) +} diff --git a/third_party/rust/flate2/examples/gzdecoder-write.rs b/third_party/rust/flate2/examples/gzdecoder-write.rs new file mode 100644 index 0000000000..766bb5e756 --- /dev/null +++ b/third_party/rust/flate2/examples/gzdecoder-write.rs @@ -0,0 +1,26 @@ +extern crate flate2; + +use flate2::write::{GzDecoder, GzEncoder}; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_writer(bytes).unwrap()); +} + +// Uncompresses a Gz Encoded vector of bytes and returns a string or error +// Here &[u8] implements Read +fn decode_writer(bytes: Vec<u8>) -> io::Result<String> { + let mut writer = Vec::new(); + let mut decoder = GzDecoder::new(writer); + decoder.write_all(&bytes[..])?; + decoder.try_finish()?; + writer = decoder.finish()?; + let return_string = String::from_utf8(writer).expect("String parsing error"); + Ok(return_string) +} diff --git a/third_party/rust/flate2/examples/gzencoder-bufread.rs b/third_party/rust/flate2/examples/gzencoder-bufread.rs new file mode 100644 index 0000000000..015ae0a82f --- /dev/null +++ b/third_party/rust/flate2/examples/gzencoder-bufread.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::bufread::GzEncoder; +use flate2::Compression; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::io::BufReader; + +// Open file and debug print the contents compressed with gzip +fn main() { + println!("{:?}", open_hello_world().unwrap()); +} + +// Opens sample file, compresses the contents and returns a Vector or error +// File wrapped in a BufReader implements Bufread +fn open_hello_world() -> io::Result<Vec<u8>> { + let f = File::open("examples/hello_world.txt")?; + let b = BufReader::new(f); + let mut gz = GzEncoder::new(b, Compression::fast()); + let mut buffer = Vec::new(); + gz.read_to_end(&mut buffer)?; + Ok(buffer) +} diff --git a/third_party/rust/flate2/examples/gzencoder-read.rs b/third_party/rust/flate2/examples/gzencoder-read.rs new file mode 100644 index 0000000000..a9657aca16 --- /dev/null +++ b/third_party/rust/flate2/examples/gzencoder-read.rs @@ -0,0 +1,20 @@ +extern crate flate2; + +use flate2::read::GzEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Print the GZ compressed representation of hello world +fn main() { + println!("{:?}", gzencoder_read_hello_world().unwrap()); +} + +// Return a vector containing the GZ compressed version of hello world +fn gzencoder_read_hello_world() -> io::Result<Vec<u8>> { + let mut ret_vec = [0; 100]; + let c = b"hello world"; + let mut z = GzEncoder::new(&c[..], Compression::fast()); + let count = z.read(&mut ret_vec)?; + Ok(ret_vec[0..count].to_vec()) +} diff --git a/third_party/rust/flate2/examples/gzencoder-write.rs b/third_party/rust/flate2/examples/gzencoder-write.rs new file mode 100644 index 0000000000..275b010c5f --- /dev/null +++ b/third_party/rust/flate2/examples/gzencoder-write.rs @@ -0,0 +1,12 @@ +extern crate flate2; + +use flate2::write::GzEncoder; +use flate2::Compression; +use std::io::prelude::*; + +// Vec<u8> implements Write to print the compressed bytes of sample string +fn main() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + println!("{:?}", e.finish().unwrap()); +} diff --git a/third_party/rust/flate2/examples/gzmultidecoder-bufread.rs b/third_party/rust/flate2/examples/gzmultidecoder-bufread.rs new file mode 100644 index 0000000000..c6bb2c5504 --- /dev/null +++ b/third_party/rust/flate2/examples/gzmultidecoder-bufread.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::bufread::MultiGzDecoder; +use flate2::write::GzEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Gz Encoded vector of bytes and returns a string or error +// Here &[u8] implements BufRead +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut gz = MultiGzDecoder::new(&bytes[..]); + let mut s = String::new(); + gz.read_to_string(&mut s)?; + Ok(s) +} diff --git a/third_party/rust/flate2/examples/gzmultidecoder-read.rs b/third_party/rust/flate2/examples/gzmultidecoder-read.rs new file mode 100644 index 0000000000..7c8a8e3168 --- /dev/null +++ b/third_party/rust/flate2/examples/gzmultidecoder-read.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::read::MultiGzDecoder; +use flate2::write::GzEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Gz Encoded vector of bytes and returns a string or error +// Here &[u8] implements Read +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut gz = MultiGzDecoder::new(&bytes[..]); + let mut s = String::new(); + gz.read_to_string(&mut s)?; + Ok(s) +} diff --git a/third_party/rust/flate2/examples/hello_world.txt b/third_party/rust/flate2/examples/hello_world.txt new file mode 100644 index 0000000000..557db03de9 --- /dev/null +++ b/third_party/rust/flate2/examples/hello_world.txt @@ -0,0 +1 @@ +Hello World diff --git a/third_party/rust/flate2/examples/zlibdecoder-bufread.rs b/third_party/rust/flate2/examples/zlibdecoder-bufread.rs new file mode 100644 index 0000000000..30f168a7b1 --- /dev/null +++ b/third_party/rust/flate2/examples/zlibdecoder-bufread.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::bufread::ZlibDecoder; +use flate2::write::ZlibEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_bufreader(bytes).unwrap()); +} + +// Uncompresses a Zlib Encoded vector of bytes and returns a string or error +// Here &[u8] implements BufRead +fn decode_bufreader(bytes: Vec<u8>) -> io::Result<String> { + let mut z = ZlibDecoder::new(&bytes[..]); + let mut s = String::new(); + z.read_to_string(&mut s)?; + Ok(s) +} diff --git a/third_party/rust/flate2/examples/zlibdecoder-read.rs b/third_party/rust/flate2/examples/zlibdecoder-read.rs new file mode 100644 index 0000000000..f7e5fb0af6 --- /dev/null +++ b/third_party/rust/flate2/examples/zlibdecoder-read.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::read::ZlibDecoder; +use flate2::write::ZlibEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Zlib Encoded vector of bytes and returns a string or error +// Here &[u8] implements Read +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut z = ZlibDecoder::new(&bytes[..]); + let mut s = String::new(); + z.read_to_string(&mut s)?; + Ok(s) +} diff --git a/third_party/rust/flate2/examples/zlibdecoder-write.rs b/third_party/rust/flate2/examples/zlibdecoder-write.rs new file mode 100644 index 0000000000..358e9035b1 --- /dev/null +++ b/third_party/rust/flate2/examples/zlibdecoder-write.rs @@ -0,0 +1,26 @@ +extern crate flate2; + +use flate2::write::ZlibDecoder; +use flate2::write::ZlibEncoder; +use flate2::Compression; +use std::io; +use std::io::prelude::*; + +// Compress a sample string and print it after transformation. +fn main() { + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + let bytes = e.finish().unwrap(); + println!("{}", decode_reader(bytes).unwrap()); +} + +// Uncompresses a Zlib Encoded vector of bytes and returns a string or error +// Here Vec<u8> implements Write +fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { + let mut writer = Vec::new(); + let mut z = ZlibDecoder::new(writer); + z.write_all(&bytes[..])?; + writer = z.finish()?; + let return_string = String::from_utf8(writer).expect("String parsing error"); + Ok(return_string) +} diff --git a/third_party/rust/flate2/examples/zlibencoder-bufread.rs b/third_party/rust/flate2/examples/zlibencoder-bufread.rs new file mode 100644 index 0000000000..0321d8d38b --- /dev/null +++ b/third_party/rust/flate2/examples/zlibencoder-bufread.rs @@ -0,0 +1,24 @@ +extern crate flate2; + +use flate2::bufread::ZlibEncoder; +use flate2::Compression; +use std::fs::File; +use std::io; +use std::io::prelude::*; +use std::io::BufReader; + +// Open file and debug print the contents compressed with zlib +fn main() { + println!("{:?}", open_hello_world().unwrap()); +} + +// Opens sample file, compresses the contents and returns a Vector or error +// File wrapped in a BufReader implements Bufread +fn open_hello_world() -> io::Result<Vec<u8>> { + let f = File::open("examples/hello_world.txt")?; + let b = BufReader::new(f); + let mut z = ZlibEncoder::new(b, Compression::fast()); + let mut buffer = Vec::new(); + z.read_to_end(&mut buffer)?; + Ok(buffer) +} diff --git a/third_party/rust/flate2/examples/zlibencoder-read.rs b/third_party/rust/flate2/examples/zlibencoder-read.rs new file mode 100644 index 0000000000..b0ae50a97c --- /dev/null +++ b/third_party/rust/flate2/examples/zlibencoder-read.rs @@ -0,0 +1,21 @@ +extern crate flate2; + +use flate2::read::ZlibEncoder; +use flate2::Compression; +use std::fs::File; +use std::io::prelude::*; + +// Open file and debug print the compressed contents +fn main() { + println!("{:?}", open_hello_world().unwrap()); +} + +// Opens sample file, compresses the contents and returns a Vector or error +// File implements Read +fn open_hello_world() -> std::io::Result<Vec<u8>> { + let f = File::open("examples/hello_world.txt")?; + let mut z = ZlibEncoder::new(f, Compression::fast()); + let mut buffer = [0; 50]; + let byte_count = z.read(&mut buffer)?; + Ok(buffer[0..byte_count].to_vec()) +} diff --git a/third_party/rust/flate2/examples/zlibencoder-write.rs b/third_party/rust/flate2/examples/zlibencoder-write.rs new file mode 100644 index 0000000000..76bcf17b17 --- /dev/null +++ b/third_party/rust/flate2/examples/zlibencoder-write.rs @@ -0,0 +1,12 @@ +extern crate flate2; + +use flate2::write::ZlibEncoder; +use flate2::Compression; +use std::io::prelude::*; + +// Vec<u8> implements Write to print the compressed bytes of sample string +fn main() { + let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"Hello World").unwrap(); + println!("{:?}", e.finish().unwrap()); +} diff --git a/third_party/rust/flate2/src/bufreader.rs b/third_party/rust/flate2/src/bufreader.rs new file mode 100644 index 0000000000..9aa6a3ae98 --- /dev/null +++ b/third_party/rust/flate2/src/bufreader.rs @@ -0,0 +1,104 @@ +// Copyright 2013 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or +// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license +// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +use std::cmp; +use std::io; +use std::io::prelude::*; +use std::mem; + +pub struct BufReader<R> { + inner: R, + buf: Box<[u8]>, + pos: usize, + cap: usize, +} + +impl<R> ::std::fmt::Debug for BufReader<R> +where + R: ::std::fmt::Debug, +{ + fn fmt(&self, fmt: &mut ::std::fmt::Formatter) -> Result<(), ::std::fmt::Error> { + fmt.debug_struct("BufReader") + .field("reader", &self.inner) + .field( + "buffer", + &format_args!("{}/{}", self.cap - self.pos, self.buf.len()), + ) + .finish() + } +} + +impl<R: Read> BufReader<R> { + pub fn new(inner: R) -> BufReader<R> { + BufReader::with_buf(vec![0; 32 * 1024], inner) + } + + pub fn with_buf(buf: Vec<u8>, inner: R) -> BufReader<R> { + BufReader { + inner: inner, + buf: buf.into_boxed_slice(), + pos: 0, + cap: 0, + } + } +} + +impl<R> BufReader<R> { + pub fn get_ref(&self) -> &R { + &self.inner + } + + pub fn get_mut(&mut self) -> &mut R { + &mut self.inner + } + + pub fn into_inner(self) -> R { + self.inner + } + + pub fn reset(&mut self, inner: R) -> R { + self.pos = 0; + self.cap = 0; + mem::replace(&mut self.inner, inner) + } +} + +impl<R: Read> Read for BufReader<R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + // If we don't have any buffered data and we're doing a massive read + // (larger than our internal buffer), bypass our internal buffer + // entirely. + if self.pos == self.cap && buf.len() >= self.buf.len() { + return self.inner.read(buf); + } + let nread = { + let mut rem = self.fill_buf()?; + rem.read(buf)? + }; + self.consume(nread); + Ok(nread) + } +} + +impl<R: Read> BufRead for BufReader<R> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + // If we've reached the end of our internal buffer then we need to fetch + // some more data from the underlying reader. + if self.pos == self.cap { + self.cap = self.inner.read(&mut self.buf)?; + self.pos = 0; + } + Ok(&self.buf[self.pos..self.cap]) + } + + fn consume(&mut self, amt: usize) { + self.pos = cmp::min(self.pos + amt, self.cap); + } +} diff --git a/third_party/rust/flate2/src/crc.rs b/third_party/rust/flate2/src/crc.rs new file mode 100644 index 0000000000..f1e20f2bd6 --- /dev/null +++ b/third_party/rust/flate2/src/crc.rs @@ -0,0 +1,178 @@ +//! Simple CRC bindings backed by miniz.c + +use std::io; +use std::io::prelude::*; + +use crc32fast::Hasher; + +/// The CRC calculated by a [`CrcReader`]. +/// +/// [`CrcReader`]: struct.CrcReader.html +#[derive(Debug)] +pub struct Crc { + amt: u32, + hasher: Hasher, +} + +/// A wrapper around a [`Read`] that calculates the CRC. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +#[derive(Debug)] +pub struct CrcReader<R> { + inner: R, + crc: Crc, +} + +impl Crc { + /// Create a new CRC. + pub fn new() -> Crc { + Crc { + amt: 0, + hasher: Hasher::new(), + } + } + + /// Returns the current crc32 checksum. + pub fn sum(&self) -> u32 { + self.hasher.clone().finalize() + } + + /// The number of bytes that have been used to calculate the CRC. + /// This value is only accurate if the amount is lower than 2<sup>32</sup>. + pub fn amount(&self) -> u32 { + self.amt + } + + /// Update the CRC with the bytes in `data`. + pub fn update(&mut self, data: &[u8]) { + self.amt = self.amt.wrapping_add(data.len() as u32); + self.hasher.update(data); + } + + /// Reset the CRC. + pub fn reset(&mut self) { + self.amt = 0; + self.hasher.reset(); + } + + /// Combine the CRC with the CRC for the subsequent block of bytes. + pub fn combine(&mut self, additional_crc: &Crc) { + self.amt += additional_crc.amt; + self.hasher.combine(&additional_crc.hasher); + } +} + +impl<R: Read> CrcReader<R> { + /// Create a new CrcReader. + pub fn new(r: R) -> CrcReader<R> { + CrcReader { + inner: r, + crc: Crc::new(), + } + } +} + +impl<R> CrcReader<R> { + /// Get the Crc for this CrcReader. + pub fn crc(&self) -> &Crc { + &self.crc + } + + /// Get the reader that is wrapped by this CrcReader. + pub fn into_inner(self) -> R { + self.inner + } + + /// Get the reader that is wrapped by this CrcReader by reference. + pub fn get_ref(&self) -> &R { + &self.inner + } + + /// Get a mutable reference to the reader that is wrapped by this CrcReader. + pub fn get_mut(&mut self) -> &mut R { + &mut self.inner + } + + /// Reset the Crc in this CrcReader. + pub fn reset(&mut self) { + self.crc.reset(); + } +} + +impl<R: Read> Read for CrcReader<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + let amt = self.inner.read(into)?; + self.crc.update(&into[..amt]); + Ok(amt) + } +} + +impl<R: BufRead> BufRead for CrcReader<R> { + fn fill_buf(&mut self) -> io::Result<&[u8]> { + self.inner.fill_buf() + } + fn consume(&mut self, amt: usize) { + if let Ok(data) = self.inner.fill_buf() { + self.crc.update(&data[..amt]); + } + self.inner.consume(amt); + } +} + +/// A wrapper around a [`Write`] that calculates the CRC. +/// +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +#[derive(Debug)] +pub struct CrcWriter<W> { + inner: W, + crc: Crc, +} + +impl<W> CrcWriter<W> { + /// Get the Crc for this CrcWriter. + pub fn crc(&self) -> &Crc { + &self.crc + } + + /// Get the writer that is wrapped by this CrcWriter. + pub fn into_inner(self) -> W { + self.inner + } + + /// Get the writer that is wrapped by this CrcWriter by reference. + pub fn get_ref(&self) -> &W { + &self.inner + } + + /// Get a mutable reference to the writer that is wrapped by this CrcWriter. + pub fn get_mut(&mut self) -> &mut W { + &mut self.inner + } + + /// Reset the Crc in this CrcWriter. + pub fn reset(&mut self) { + self.crc.reset(); + } +} + +impl<W: Write> CrcWriter<W> { + /// Create a new CrcWriter. + pub fn new(w: W) -> CrcWriter<W> { + CrcWriter { + inner: w, + crc: Crc::new(), + } + } +} + +impl<W: Write> Write for CrcWriter<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + let amt = self.inner.write(buf)?; + self.crc.update(&buf[..amt]); + Ok(amt) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} diff --git a/third_party/rust/flate2/src/deflate/bufread.rs b/third_party/rust/flate2/src/deflate/bufread.rs new file mode 100644 index 0000000000..98aee70b04 --- /dev/null +++ b/third_party/rust/flate2/src/deflate/bufread.rs @@ -0,0 +1,268 @@ +use std::io; +use std::io::prelude::*; +use std::mem; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use crate::zio; +use crate::{Compress, Decompress}; + +/// A DEFLATE encoder, or compressor. +/// +/// This structure consumes a [`BufRead`] interface, reading uncompressed data +/// from the underlying reader, and emitting compressed data. +/// +/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// use flate2::Compression; +/// use flate2::bufread::DeflateEncoder; +/// use std::fs::File; +/// use std::io::BufReader; +/// +/// # fn main() { +/// # println!("{:?}", open_hello_world().unwrap()); +/// # } +/// # +/// // Opens sample file, compresses the contents and returns a Vector +/// fn open_hello_world() -> io::Result<Vec<u8>> { +/// let f = File::open("examples/hello_world.txt")?; +/// let b = BufReader::new(f); +/// let mut deflater = DeflateEncoder::new(b, Compression::fast()); +/// let mut buffer = Vec::new(); +/// deflater.read_to_end(&mut buffer)?; +/// Ok(buffer) +/// } +/// ``` +#[derive(Debug)] +pub struct DeflateEncoder<R> { + obj: R, + data: Compress, +} + +impl<R: BufRead> DeflateEncoder<R> { + /// Creates a new encoder which will read uncompressed data from the given + /// stream and emit the compressed stream. + pub fn new(r: R, level: crate::Compression) -> DeflateEncoder<R> { + DeflateEncoder { + obj: r, + data: Compress::new(level, false), + } + } +} + +pub fn reset_encoder_data<R>(zlib: &mut DeflateEncoder<R>) { + zlib.data.reset(); +} + +impl<R> DeflateEncoder<R> { + /// Resets the state of this encoder entirely, swapping out the input + /// stream for another. + /// + /// This function will reset the internal state of this encoder and replace + /// the input stream with the one provided, returning the previous input + /// stream. Future data read from this encoder will be the compressed + /// version of `r`'s data. + pub fn reset(&mut self, r: R) -> R { + reset_encoder_data(self); + mem::replace(&mut self.obj, r) + } + + /// Acquires a reference to the underlying reader + pub fn get_ref(&self) -> &R { + &self.obj + } + + /// Acquires a mutable reference to the underlying stream + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + &mut self.obj + } + + /// Consumes this encoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.obj + } + + /// Returns the number of bytes that have been read into this compressor. + /// + /// Note that not all bytes read from the underlying object may be accounted + /// for, there may still be some active buffering. + pub fn total_in(&self) -> u64 { + self.data.total_in() + } + + /// Returns the number of bytes that the compressor has produced. + /// + /// Note that not all bytes may have been read yet, some may still be + /// buffered. + pub fn total_out(&self) -> u64 { + self.data.total_out() + } +} + +impl<R: BufRead> Read for DeflateEncoder<R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + zio::read(&mut self.obj, &mut self.data, buf) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + BufRead> AsyncRead for DeflateEncoder<R> {} + +impl<W: BufRead + Write> Write for DeflateEncoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + BufRead> AsyncWrite for DeflateEncoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} + +/// A DEFLATE decoder, or decompressor. +/// +/// This structure consumes a [`BufRead`] interface, reading compressed data +/// from the underlying reader, and emitting uncompressed data. +/// +/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::DeflateEncoder; +/// use flate2::bufread::DeflateDecoder; +/// +/// # fn main() { +/// # let mut e = DeflateEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_reader(bytes).unwrap()); +/// # } +/// // Uncompresses a Deflate Encoded vector of bytes and returns a string or error +/// // Here &[u8] implements Read +/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut deflater = DeflateDecoder::new(&bytes[..]); +/// let mut s = String::new(); +/// deflater.read_to_string(&mut s)?; +/// Ok(s) +/// } +/// ``` +#[derive(Debug)] +pub struct DeflateDecoder<R> { + obj: R, + data: Decompress, +} + +pub fn reset_decoder_data<R>(zlib: &mut DeflateDecoder<R>) { + zlib.data = Decompress::new(false); +} + +impl<R: BufRead> DeflateDecoder<R> { + /// Creates a new decoder which will decompress data read from the given + /// stream. + pub fn new(r: R) -> DeflateDecoder<R> { + DeflateDecoder { + obj: r, + data: Decompress::new(false), + } + } +} + +impl<R> DeflateDecoder<R> { + /// Resets the state of this decoder entirely, swapping out the input + /// stream for another. + /// + /// This will reset the internal state of this decoder and replace the + /// input stream with the one provided, returning the previous input + /// stream. Future data read from this decoder will be the decompressed + /// version of `r`'s data. + pub fn reset(&mut self, r: R) -> R { + reset_decoder_data(self); + mem::replace(&mut self.obj, r) + } + + /// Resets the state of this decoder's data + /// + /// This will reset the internal state of this decoder. It will continue + /// reading from the same stream. + pub fn reset_data(&mut self) { + reset_decoder_data(self); + } + + /// Acquires a reference to the underlying stream + pub fn get_ref(&self) -> &R { + &self.obj + } + + /// Acquires a mutable reference to the underlying stream + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + &mut self.obj + } + + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.obj + } + + /// Returns the number of bytes that the decompressor has consumed. + /// + /// Note that this will likely be smaller than what the decompressor + /// actually read from the underlying stream due to buffering. + pub fn total_in(&self) -> u64 { + self.data.total_in() + } + + /// Returns the number of bytes that the decompressor has produced. + pub fn total_out(&self) -> u64 { + self.data.total_out() + } +} + +impl<R: BufRead> Read for DeflateDecoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + zio::read(&mut self.obj, &mut self.data, into) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + BufRead> AsyncRead for DeflateDecoder<R> {} + +impl<W: BufRead + Write> Write for DeflateDecoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + BufRead> AsyncWrite for DeflateDecoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} diff --git a/third_party/rust/flate2/src/deflate/mod.rs b/third_party/rust/flate2/src/deflate/mod.rs new file mode 100644 index 0000000000..90a4241781 --- /dev/null +++ b/third_party/rust/flate2/src/deflate/mod.rs @@ -0,0 +1,193 @@ +pub mod bufread; +pub mod read; +pub mod write; + +#[cfg(test)] +mod tests { + use std::io::prelude::*; + + use rand::{thread_rng, Rng}; + + use super::{read, write}; + use crate::Compression; + + #[test] + fn roundtrip() { + let mut real = Vec::new(); + let mut w = write::DeflateEncoder::new(Vec::new(), Compression::default()); + let v = crate::random_bytes().take(1024).collect::<Vec<_>>(); + for _ in 0..200 { + let to_write = &v[..thread_rng().gen_range(0, v.len())]; + real.extend(to_write.iter().map(|x| *x)); + w.write_all(to_write).unwrap(); + } + let result = w.finish().unwrap(); + let mut r = read::DeflateDecoder::new(&result[..]); + let mut ret = Vec::new(); + r.read_to_end(&mut ret).unwrap(); + assert!(ret == real); + } + + #[test] + fn drop_writes() { + let mut data = Vec::new(); + write::DeflateEncoder::new(&mut data, Compression::default()) + .write_all(b"foo") + .unwrap(); + let mut r = read::DeflateDecoder::new(&data[..]); + let mut ret = Vec::new(); + r.read_to_end(&mut ret).unwrap(); + assert!(ret == b"foo"); + } + + #[test] + fn total_in() { + let mut real = Vec::new(); + let mut w = write::DeflateEncoder::new(Vec::new(), Compression::default()); + let v = crate::random_bytes().take(1024).collect::<Vec<_>>(); + for _ in 0..200 { + let to_write = &v[..thread_rng().gen_range(0, v.len())]; + real.extend(to_write.iter().map(|x| *x)); + w.write_all(to_write).unwrap(); + } + let mut result = w.finish().unwrap(); + + let result_len = result.len(); + + for _ in 0..200 { + result.extend(v.iter().map(|x| *x)); + } + + let mut r = read::DeflateDecoder::new(&result[..]); + let mut ret = Vec::new(); + r.read_to_end(&mut ret).unwrap(); + assert!(ret == real); + assert_eq!(r.total_in(), result_len as u64); + } + + #[test] + fn roundtrip2() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let mut r = + read::DeflateDecoder::new(read::DeflateEncoder::new(&v[..], Compression::default())); + let mut ret = Vec::new(); + r.read_to_end(&mut ret).unwrap(); + assert_eq!(ret, v); + } + + #[test] + fn roundtrip3() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let mut w = write::DeflateEncoder::new( + write::DeflateDecoder::new(Vec::new()), + Compression::default(), + ); + w.write_all(&v).unwrap(); + let w = w.finish().unwrap().finish().unwrap(); + assert!(w == v); + } + + #[test] + fn reset_writer() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let mut w = write::DeflateEncoder::new(Vec::new(), Compression::default()); + w.write_all(&v).unwrap(); + let a = w.reset(Vec::new()).unwrap(); + w.write_all(&v).unwrap(); + let b = w.finish().unwrap(); + + let mut w = write::DeflateEncoder::new(Vec::new(), Compression::default()); + w.write_all(&v).unwrap(); + let c = w.finish().unwrap(); + assert!(a == b && b == c); + } + + #[test] + fn reset_reader() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let (mut a, mut b, mut c) = (Vec::new(), Vec::new(), Vec::new()); + let mut r = read::DeflateEncoder::new(&v[..], Compression::default()); + r.read_to_end(&mut a).unwrap(); + r.reset(&v[..]); + r.read_to_end(&mut b).unwrap(); + + let mut r = read::DeflateEncoder::new(&v[..], Compression::default()); + r.read_to_end(&mut c).unwrap(); + assert!(a == b && b == c); + } + + #[test] + fn reset_decoder() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let mut w = write::DeflateEncoder::new(Vec::new(), Compression::default()); + w.write_all(&v).unwrap(); + let data = w.finish().unwrap(); + + { + let (mut a, mut b, mut c) = (Vec::new(), Vec::new(), Vec::new()); + let mut r = read::DeflateDecoder::new(&data[..]); + r.read_to_end(&mut a).unwrap(); + r.reset(&data); + r.read_to_end(&mut b).unwrap(); + + let mut r = read::DeflateDecoder::new(&data[..]); + r.read_to_end(&mut c).unwrap(); + assert!(a == b && b == c && c == v); + } + + { + let mut w = write::DeflateDecoder::new(Vec::new()); + w.write_all(&data).unwrap(); + let a = w.reset(Vec::new()).unwrap(); + w.write_all(&data).unwrap(); + let b = w.finish().unwrap(); + + let mut w = write::DeflateDecoder::new(Vec::new()); + w.write_all(&data).unwrap(); + let c = w.finish().unwrap(); + assert!(a == b && b == c && c == v); + } + } + + #[test] + fn zero_length_read_with_data() { + let m = vec![3u8; 128 * 1024 + 1]; + let mut c = read::DeflateEncoder::new(&m[..], Compression::default()); + + let mut result = Vec::new(); + c.read_to_end(&mut result).unwrap(); + + let mut d = read::DeflateDecoder::new(&result[..]); + let mut data = Vec::new(); + assert!(d.read(&mut data).unwrap() == 0); + } + + #[test] + fn qc_reader() { + ::quickcheck::quickcheck(test as fn(_) -> _); + + fn test(v: Vec<u8>) -> bool { + let mut r = read::DeflateDecoder::new(read::DeflateEncoder::new( + &v[..], + Compression::default(), + )); + let mut v2 = Vec::new(); + r.read_to_end(&mut v2).unwrap(); + v == v2 + } + } + + #[test] + fn qc_writer() { + ::quickcheck::quickcheck(test as fn(_) -> _); + + fn test(v: Vec<u8>) -> bool { + let mut w = write::DeflateEncoder::new( + write::DeflateDecoder::new(Vec::new()), + Compression::default(), + ); + w.write_all(&v).unwrap(); + v == w.finish().unwrap().finish().unwrap() + } + } +} diff --git a/third_party/rust/flate2/src/deflate/read.rs b/third_party/rust/flate2/src/deflate/read.rs new file mode 100644 index 0000000000..21748e72c3 --- /dev/null +++ b/third_party/rust/flate2/src/deflate/read.rs @@ -0,0 +1,266 @@ +use std::io; +use std::io::prelude::*; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use super::bufread; +use crate::bufreader::BufReader; + +/// A DEFLATE encoder, or compressor. +/// +/// This structure implements a [`Read`] interface and will read uncompressed +/// data from an underlying stream and emit a stream of compressed data. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// use flate2::Compression; +/// use flate2::read::DeflateEncoder; +/// +/// # fn main() { +/// # println!("{:?}", deflateencoder_read_hello_world().unwrap()); +/// # } +/// # +/// // Return a vector containing the Deflate compressed version of hello world +/// fn deflateencoder_read_hello_world() -> io::Result<Vec<u8>> { +/// let mut ret_vec = [0;100]; +/// let c = b"hello world"; +/// let mut deflater = DeflateEncoder::new(&c[..], Compression::fast()); +/// let count = deflater.read(&mut ret_vec)?; +/// Ok(ret_vec[0..count].to_vec()) +/// } +/// ``` +#[derive(Debug)] +pub struct DeflateEncoder<R> { + inner: bufread::DeflateEncoder<BufReader<R>>, +} + +impl<R: Read> DeflateEncoder<R> { + /// Creates a new encoder which will read uncompressed data from the given + /// stream and emit the compressed stream. + pub fn new(r: R, level: crate::Compression) -> DeflateEncoder<R> { + DeflateEncoder { + inner: bufread::DeflateEncoder::new(BufReader::new(r), level), + } + } +} + +impl<R> DeflateEncoder<R> { + /// Resets the state of this encoder entirely, swapping out the input + /// stream for another. + /// + /// This function will reset the internal state of this encoder and replace + /// the input stream with the one provided, returning the previous input + /// stream. Future data read from this encoder will be the compressed + /// version of `r`'s data. + /// + /// Note that there may be currently buffered data when this function is + /// called, and in that case the buffered data is discarded. + pub fn reset(&mut self, r: R) -> R { + super::bufread::reset_encoder_data(&mut self.inner); + self.inner.get_mut().reset(r) + } + + /// Acquires a reference to the underlying reader + pub fn get_ref(&self) -> &R { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying stream + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.inner.get_mut().get_mut() + } + + /// Consumes this encoder, returning the underlying reader. + /// + /// Note that there may be buffered bytes which are not re-acquired as part + /// of this transition. It's recommended to only call this function after + /// EOF has been reached. + pub fn into_inner(self) -> R { + self.inner.into_inner().into_inner() + } + + /// Returns the number of bytes that have been read into this compressor. + /// + /// Note that not all bytes read from the underlying object may be accounted + /// for, there may still be some active buffering. + pub fn total_in(&self) -> u64 { + self.inner.total_in() + } + + /// Returns the number of bytes that the compressor has produced. + /// + /// Note that not all bytes may have been read yet, some may still be + /// buffered. + pub fn total_out(&self) -> u64 { + self.inner.total_out() + } +} + +impl<R: Read> Read for DeflateEncoder<R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.inner.read(buf) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead> AsyncRead for DeflateEncoder<R> {} + +impl<W: Read + Write> Write for DeflateEncoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + AsyncWrite> AsyncWrite for DeflateEncoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} + +/// A DEFLATE decoder, or decompressor. +/// +/// This structure implements a [`Read`] interface and takes a stream of +/// compressed data as input, providing the decompressed data when read from. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::DeflateEncoder; +/// use flate2::read::DeflateDecoder; +/// +/// # fn main() { +/// # let mut e = DeflateEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_reader(bytes).unwrap()); +/// # } +/// // Uncompresses a Deflate Encoded vector of bytes and returns a string or error +/// // Here &[u8] implements Read +/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut deflater = DeflateDecoder::new(&bytes[..]); +/// let mut s = String::new(); +/// deflater.read_to_string(&mut s)?; +/// Ok(s) +/// } +/// ``` +#[derive(Debug)] +pub struct DeflateDecoder<R> { + inner: bufread::DeflateDecoder<BufReader<R>>, +} + +impl<R: Read> DeflateDecoder<R> { + /// Creates a new decoder which will decompress data read from the given + /// stream. + pub fn new(r: R) -> DeflateDecoder<R> { + DeflateDecoder::new_with_buf(r, vec![0; 32 * 1024]) + } + + /// Same as `new`, but the intermediate buffer for data is specified. + /// + /// Note that the capacity of the intermediate buffer is never increased, + /// and it is recommended for it to be large. + pub fn new_with_buf(r: R, buf: Vec<u8>) -> DeflateDecoder<R> { + DeflateDecoder { + inner: bufread::DeflateDecoder::new(BufReader::with_buf(buf, r)), + } + } +} + +impl<R> DeflateDecoder<R> { + /// Resets the state of this decoder entirely, swapping out the input + /// stream for another. + /// + /// This will reset the internal state of this decoder and replace the + /// input stream with the one provided, returning the previous input + /// stream. Future data read from this decoder will be the decompressed + /// version of `r`'s data. + /// + /// Note that there may be currently buffered data when this function is + /// called, and in that case the buffered data is discarded. + pub fn reset(&mut self, r: R) -> R { + super::bufread::reset_decoder_data(&mut self.inner); + self.inner.get_mut().reset(r) + } + + /// Acquires a reference to the underlying stream + pub fn get_ref(&self) -> &R { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying stream + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.inner.get_mut().get_mut() + } + + /// Consumes this decoder, returning the underlying reader. + /// + /// Note that there may be buffered bytes which are not re-acquired as part + /// of this transition. It's recommended to only call this function after + /// EOF has been reached. + pub fn into_inner(self) -> R { + self.inner.into_inner().into_inner() + } + + /// Returns the number of bytes that the decompressor has consumed. + /// + /// Note that this will likely be smaller than what the decompressor + /// actually read from the underlying stream due to buffering. + pub fn total_in(&self) -> u64 { + self.inner.total_in() + } + + /// Returns the number of bytes that the decompressor has produced. + pub fn total_out(&self) -> u64 { + self.inner.total_out() + } +} + +impl<R: Read> Read for DeflateDecoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + self.inner.read(into) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead> AsyncRead for DeflateDecoder<R> {} + +impl<W: Read + Write> Write for DeflateDecoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + AsyncRead> AsyncWrite for DeflateDecoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} diff --git a/third_party/rust/flate2/src/deflate/write.rs b/third_party/rust/flate2/src/deflate/write.rs new file mode 100644 index 0000000000..a9aadb98fe --- /dev/null +++ b/third_party/rust/flate2/src/deflate/write.rs @@ -0,0 +1,349 @@ +use std::io; +use std::io::prelude::*; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use crate::zio; +use crate::{Compress, Decompress}; + +/// A DEFLATE encoder, or compressor. +/// +/// This structure implements a [`Write`] interface and takes a stream of +/// uncompressed data, writing the compressed data to the wrapped writer. +/// +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use flate2::Compression; +/// use flate2::write::DeflateEncoder; +/// +/// // Vec<u8> implements Write to print the compressed bytes of sample string +/// # fn main() { +/// +/// let mut e = DeflateEncoder::new(Vec::new(), Compression::default()); +/// e.write_all(b"Hello World").unwrap(); +/// println!("{:?}", e.finish().unwrap()); +/// # } +/// ``` +#[derive(Debug)] +pub struct DeflateEncoder<W: Write> { + inner: zio::Writer<W, Compress>, +} + +impl<W: Write> DeflateEncoder<W> { + /// Creates a new encoder which will write compressed data to the stream + /// given at the given compression level. + /// + /// When this encoder is dropped or unwrapped the final pieces of data will + /// be flushed. + pub fn new(w: W, level: crate::Compression) -> DeflateEncoder<W> { + DeflateEncoder { + inner: zio::Writer::new(w, Compress::new(level, false)), + } + } + + /// Acquires a reference to the underlying writer. + pub fn get_ref(&self) -> &W { + self.inner.get_ref() + } + + /// Acquires a mutable reference to the underlying writer. + /// + /// Note that mutating the output/input state of the stream may corrupt this + /// object, so care must be taken when using this method. + pub fn get_mut(&mut self) -> &mut W { + self.inner.get_mut() + } + + /// Resets the state of this encoder entirely, swapping out the output + /// stream for another. + /// + /// This function will finish encoding the current stream into the current + /// output stream before swapping out the two output streams. If the stream + /// cannot be finished an error is returned. + /// + /// After the current stream has been finished, this will reset the internal + /// state of this encoder and replace the output stream with the one + /// provided, returning the previous output stream. Future data written to + /// this encoder will be the compressed into the stream `w` provided. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn reset(&mut self, w: W) -> io::Result<W> { + self.inner.finish()?; + self.inner.data.reset(); + Ok(self.inner.replace(w)) + } + + /// Attempt to finish this output stream, writing out final chunks of data. + /// + /// Note that this function can only be used once data has finished being + /// written to the output stream. After this function is called then further + /// calls to `write` may result in a panic. + /// + /// # Panics + /// + /// Attempts to write data to this stream may result in a panic after this + /// function is called. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn try_finish(&mut self) -> io::Result<()> { + self.inner.finish() + } + + /// Consumes this encoder, flushing the output stream. + /// + /// This will flush the underlying data stream, close off the compressed + /// stream and, if successful, return the contained writer. + /// + /// Note that this function may not be suitable to call in a situation where + /// the underlying stream is an asynchronous I/O stream. To finish a stream + /// the `try_finish` (or `shutdown`) method should be used instead. To + /// re-acquire ownership of a stream it is safe to call this method after + /// `try_finish` or `shutdown` has returned `Ok`. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn finish(mut self) -> io::Result<W> { + self.inner.finish()?; + Ok(self.inner.take_inner()) + } + + /// Consumes this encoder, flushing the output stream. + /// + /// This will flush the underlying data stream and then return the contained + /// writer if the flush succeeded. + /// The compressed stream will not closed but only flushed. This + /// means that obtained byte array can by extended by another deflated + /// stream. To close the stream add the two bytes 0x3 and 0x0. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn flush_finish(mut self) -> io::Result<W> { + self.inner.flush()?; + Ok(self.inner.take_inner()) + } + + /// Returns the number of bytes that have been written to this compresor. + /// + /// Note that not all bytes written to this object may be accounted for, + /// there may still be some active buffering. + pub fn total_in(&self) -> u64 { + self.inner.data.total_in() + } + + /// Returns the number of bytes that the compressor has produced. + /// + /// Note that not all bytes may have been written yet, some may still be + /// buffered. + pub fn total_out(&self) -> u64 { + self.inner.data.total_out() + } +} + +impl<W: Write> Write for DeflateEncoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.inner.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncWrite> AsyncWrite for DeflateEncoder<W> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.inner.finish()?; + self.inner.get_mut().shutdown() + } +} + +impl<W: Read + Write> Read for DeflateEncoder<W> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.inner.get_mut().read(buf) + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncRead + AsyncWrite> AsyncRead for DeflateEncoder<W> {} + +/// A DEFLATE decoder, or decompressor. +/// +/// This structure implements a [`Write`] and will emit a stream of decompressed +/// data when fed a stream of compressed data. +/// +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::DeflateEncoder; +/// use flate2::write::DeflateDecoder; +/// +/// # fn main() { +/// # let mut e = DeflateEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_writer(bytes).unwrap()); +/// # } +/// // Uncompresses a Deflate Encoded vector of bytes and returns a string or error +/// // Here Vec<u8> implements Write +/// fn decode_writer(bytes: Vec<u8>) -> io::Result<String> { +/// let mut writer = Vec::new(); +/// let mut deflater = DeflateDecoder::new(writer); +/// deflater.write_all(&bytes[..])?; +/// writer = deflater.finish()?; +/// let return_string = String::from_utf8(writer).expect("String parsing error"); +/// Ok(return_string) +/// } +/// ``` +#[derive(Debug)] +pub struct DeflateDecoder<W: Write> { + inner: zio::Writer<W, Decompress>, +} + +impl<W: Write> DeflateDecoder<W> { + /// Creates a new decoder which will write uncompressed data to the stream. + /// + /// When this encoder is dropped or unwrapped the final pieces of data will + /// be flushed. + pub fn new(w: W) -> DeflateDecoder<W> { + DeflateDecoder { + inner: zio::Writer::new(w, Decompress::new(false)), + } + } + + /// Acquires a reference to the underlying writer. + pub fn get_ref(&self) -> &W { + self.inner.get_ref() + } + + /// Acquires a mutable reference to the underlying writer. + /// + /// Note that mutating the output/input state of the stream may corrupt this + /// object, so care must be taken when using this method. + pub fn get_mut(&mut self) -> &mut W { + self.inner.get_mut() + } + + /// Resets the state of this decoder entirely, swapping out the output + /// stream for another. + /// + /// This function will finish encoding the current stream into the current + /// output stream before swapping out the two output streams. + /// + /// This will then reset the internal state of this decoder and replace the + /// output stream with the one provided, returning the previous output + /// stream. Future data written to this decoder will be decompressed into + /// the output stream `w`. + /// + /// # Errors + /// + /// This function will perform I/O to finish the stream, and if that I/O + /// returns an error then that will be returned from this function. + pub fn reset(&mut self, w: W) -> io::Result<W> { + self.inner.finish()?; + self.inner.data = Decompress::new(false); + Ok(self.inner.replace(w)) + } + + /// Attempt to finish this output stream, writing out final chunks of data. + /// + /// Note that this function can only be used once data has finished being + /// written to the output stream. After this function is called then further + /// calls to `write` may result in a panic. + /// + /// # Panics + /// + /// Attempts to write data to this stream may result in a panic after this + /// function is called. + /// + /// # Errors + /// + /// This function will perform I/O to finish the stream, returning any + /// errors which happen. + pub fn try_finish(&mut self) -> io::Result<()> { + self.inner.finish() + } + + /// Consumes this encoder, flushing the output stream. + /// + /// This will flush the underlying data stream and then return the contained + /// writer if the flush succeeded. + /// + /// Note that this function may not be suitable to call in a situation where + /// the underlying stream is an asynchronous I/O stream. To finish a stream + /// the `try_finish` (or `shutdown`) method should be used instead. To + /// re-acquire ownership of a stream it is safe to call this method after + /// `try_finish` or `shutdown` has returned `Ok`. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn finish(mut self) -> io::Result<W> { + self.inner.finish()?; + Ok(self.inner.take_inner()) + } + + /// Returns the number of bytes that the decompressor has consumed for + /// decompression. + /// + /// Note that this will likely be smaller than the number of bytes + /// successfully written to this stream due to internal buffering. + pub fn total_in(&self) -> u64 { + self.inner.data.total_in() + } + + /// Returns the number of bytes that the decompressor has written to its + /// output stream. + pub fn total_out(&self) -> u64 { + self.inner.data.total_out() + } +} + +impl<W: Write> Write for DeflateDecoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.inner.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncWrite> AsyncWrite for DeflateDecoder<W> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.inner.finish()?; + self.inner.get_mut().shutdown() + } +} + +impl<W: Read + Write> Read for DeflateDecoder<W> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.inner.get_mut().read(buf) + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncRead + AsyncWrite> AsyncRead for DeflateDecoder<W> {} diff --git a/third_party/rust/flate2/src/ffi/c.rs b/third_party/rust/flate2/src/ffi/c.rs new file mode 100644 index 0000000000..8eae160bd1 --- /dev/null +++ b/third_party/rust/flate2/src/ffi/c.rs @@ -0,0 +1,476 @@ +//! Implementation for C backends. +use std::alloc::{self, Layout}; +use std::cmp; +use std::convert::TryFrom; +use std::fmt; +use std::marker; +use std::ops::{Deref, DerefMut}; +use std::ptr; + +pub use libc::{c_int, c_uint, c_void, size_t}; + +use super::*; +use crate::mem::{self, FlushDecompress, Status}; + +pub struct StreamWrapper { + pub inner: Box<mz_stream>, +} + +impl fmt::Debug for StreamWrapper { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!(f, "StreamWrapper") + } +} + +impl Default for StreamWrapper { + fn default() -> StreamWrapper { + StreamWrapper { + inner: Box::new(mz_stream { + next_in: ptr::null_mut(), + avail_in: 0, + total_in: 0, + next_out: ptr::null_mut(), + avail_out: 0, + total_out: 0, + msg: ptr::null_mut(), + adler: 0, + data_type: 0, + reserved: 0, + opaque: ptr::null_mut(), + state: ptr::null_mut(), + #[cfg(feature = "any_zlib")] + zalloc, + #[cfg(feature = "any_zlib")] + zfree, + #[cfg(not(feature = "any_zlib"))] + zalloc: Some(zalloc), + #[cfg(not(feature = "any_zlib"))] + zfree: Some(zfree), + }), + } + } +} + +const ALIGN: usize = std::mem::align_of::<usize>(); + +fn align_up(size: usize, align: usize) -> usize { + (size + align - 1) & !(align - 1) +} + +extern "C" fn zalloc(_ptr: *mut c_void, items: AllocSize, item_size: AllocSize) -> *mut c_void { + // We need to multiply `items` and `item_size` to get the actual desired + // allocation size. Since `zfree` doesn't receive a size argument we + // also need to allocate space for a `usize` as a header so we can store + // how large the allocation is to deallocate later. + let size = match items + .checked_mul(item_size) + .and_then(|i| usize::try_from(i).ok()) + .map(|size| align_up(size, ALIGN)) + .and_then(|i| i.checked_add(std::mem::size_of::<usize>())) + { + Some(i) => i, + None => return ptr::null_mut(), + }; + + // Make sure the `size` isn't too big to fail `Layout`'s restrictions + let layout = match Layout::from_size_align(size, ALIGN) { + Ok(layout) => layout, + Err(_) => return ptr::null_mut(), + }; + + unsafe { + // Allocate the data, and if successful store the size we allocated + // at the beginning and then return an offset pointer. + let ptr = alloc::alloc(layout) as *mut usize; + if ptr.is_null() { + return ptr as *mut c_void; + } + *ptr = size; + ptr.add(1) as *mut c_void + } +} + +extern "C" fn zfree(_ptr: *mut c_void, address: *mut c_void) { + unsafe { + // Move our address being free'd back one pointer, read the size we + // stored in `zalloc`, and then free it using the standard Rust + // allocator. + let ptr = (address as *mut usize).offset(-1); + let size = *ptr; + let layout = Layout::from_size_align_unchecked(size, ALIGN); + alloc::dealloc(ptr as *mut u8, layout) + } +} + +impl Deref for StreamWrapper { + type Target = mz_stream; + + fn deref(&self) -> &Self::Target { + &*self.inner + } +} + +impl DerefMut for StreamWrapper { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut *self.inner + } +} + +unsafe impl<D: Direction> Send for Stream<D> {} +unsafe impl<D: Direction> Sync for Stream<D> {} + +/// Trait used to call the right destroy/end function on the inner +/// stream object on drop. +pub trait Direction { + unsafe fn destroy(stream: *mut mz_stream) -> c_int; +} + +#[derive(Debug)] +pub enum DirCompress {} +#[derive(Debug)] +pub enum DirDecompress {} + +#[derive(Debug)] +pub struct Stream<D: Direction> { + pub stream_wrapper: StreamWrapper, + pub total_in: u64, + pub total_out: u64, + pub _marker: marker::PhantomData<D>, +} + +impl<D: Direction> Drop for Stream<D> { + fn drop(&mut self) { + unsafe { + let _ = D::destroy(&mut *self.stream_wrapper); + } + } +} + +impl Direction for DirCompress { + unsafe fn destroy(stream: *mut mz_stream) -> c_int { + mz_deflateEnd(stream) + } +} +impl Direction for DirDecompress { + unsafe fn destroy(stream: *mut mz_stream) -> c_int { + mz_inflateEnd(stream) + } +} + +#[derive(Debug)] +pub struct Inflate { + pub inner: Stream<DirDecompress>, +} + +impl InflateBackend for Inflate { + fn make(zlib_header: bool, window_bits: u8) -> Self { + assert!( + window_bits > 8 && window_bits < 16, + "window_bits must be within 9 ..= 15" + ); + unsafe { + let mut state = StreamWrapper::default(); + let ret = mz_inflateInit2( + &mut *state, + if zlib_header { + window_bits as c_int + } else { + -(window_bits as c_int) + }, + ); + assert_eq!(ret, 0); + Inflate { + inner: Stream { + stream_wrapper: state, + total_in: 0, + total_out: 0, + _marker: marker::PhantomData, + }, + } + } + } + + fn decompress( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushDecompress, + ) -> Result<Status, DecompressError> { + let raw = &mut *self.inner.stream_wrapper; + raw.next_in = input.as_ptr() as *mut u8; + raw.avail_in = cmp::min(input.len(), c_uint::max_value() as usize) as c_uint; + raw.next_out = output.as_mut_ptr(); + raw.avail_out = cmp::min(output.len(), c_uint::max_value() as usize) as c_uint; + + let rc = unsafe { mz_inflate(raw, flush as c_int) }; + + // Unfortunately the total counters provided by zlib might be only + // 32 bits wide and overflow while processing large amounts of data. + self.inner.total_in += (raw.next_in as usize - input.as_ptr() as usize) as u64; + self.inner.total_out += (raw.next_out as usize - output.as_ptr() as usize) as u64; + + match rc { + MZ_DATA_ERROR | MZ_STREAM_ERROR => mem::decompress_failed(), + MZ_OK => Ok(Status::Ok), + MZ_BUF_ERROR => Ok(Status::BufError), + MZ_STREAM_END => Ok(Status::StreamEnd), + MZ_NEED_DICT => mem::decompress_need_dict(raw.adler as u32), + c => panic!("unknown return code: {}", c), + } + } + + #[cfg(feature = "any_zlib")] + fn reset(&mut self, zlib_header: bool) { + let bits = if zlib_header { + MZ_DEFAULT_WINDOW_BITS + } else { + -MZ_DEFAULT_WINDOW_BITS + }; + unsafe { + inflateReset2(&mut *self.inner.stream_wrapper, bits); + } + self.inner.total_out = 0; + self.inner.total_in = 0; + } + + #[cfg(not(feature = "any_zlib"))] + fn reset(&mut self, zlib_header: bool) { + *self = Self::make(zlib_header, MZ_DEFAULT_WINDOW_BITS as u8); + } +} + +impl Backend for Inflate { + #[inline] + fn total_in(&self) -> u64 { + self.inner.total_in + } + + #[inline] + fn total_out(&self) -> u64 { + self.inner.total_out + } +} + +#[derive(Debug)] +pub struct Deflate { + pub inner: Stream<DirCompress>, +} + +impl DeflateBackend for Deflate { + fn make(level: Compression, zlib_header: bool, window_bits: u8) -> Self { + assert!( + window_bits > 8 && window_bits < 16, + "window_bits must be within 9 ..= 15" + ); + unsafe { + let mut state = StreamWrapper::default(); + let ret = mz_deflateInit2( + &mut *state, + level.0 as c_int, + MZ_DEFLATED, + if zlib_header { + window_bits as c_int + } else { + -(window_bits as c_int) + }, + 8, + MZ_DEFAULT_STRATEGY, + ); + assert_eq!(ret, 0); + Deflate { + inner: Stream { + stream_wrapper: state, + total_in: 0, + total_out: 0, + _marker: marker::PhantomData, + }, + } + } + } + fn compress( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushCompress, + ) -> Result<Status, CompressError> { + let raw = &mut *self.inner.stream_wrapper; + raw.next_in = input.as_ptr() as *mut _; + raw.avail_in = cmp::min(input.len(), c_uint::max_value() as usize) as c_uint; + raw.next_out = output.as_mut_ptr(); + raw.avail_out = cmp::min(output.len(), c_uint::max_value() as usize) as c_uint; + + let rc = unsafe { mz_deflate(raw, flush as c_int) }; + + // Unfortunately the total counters provided by zlib might be only + // 32 bits wide and overflow while processing large amounts of data. + self.inner.total_in += (raw.next_in as usize - input.as_ptr() as usize) as u64; + self.inner.total_out += (raw.next_out as usize - output.as_ptr() as usize) as u64; + + match rc { + MZ_OK => Ok(Status::Ok), + MZ_BUF_ERROR => Ok(Status::BufError), + MZ_STREAM_END => Ok(Status::StreamEnd), + MZ_STREAM_ERROR => Err(CompressError(())), + c => panic!("unknown return code: {}", c), + } + } + + fn reset(&mut self) { + self.inner.total_in = 0; + self.inner.total_out = 0; + let rc = unsafe { mz_deflateReset(&mut *self.inner.stream_wrapper) }; + assert_eq!(rc, MZ_OK); + } +} + +impl Backend for Deflate { + #[inline] + fn total_in(&self) -> u64 { + self.inner.total_in + } + + #[inline] + fn total_out(&self) -> u64 { + self.inner.total_out + } +} + +pub use self::c_backend::*; + +/// Miniz specific +#[cfg(not(feature = "any_zlib"))] +mod c_backend { + pub use miniz_sys::*; + pub type AllocSize = libc::size_t; +} + +/// Zlib specific +#[cfg(any( + feature = "zlib-ng-compat", + all(feature = "zlib", not(feature = "cloudflare_zlib")) +))] +#[allow(bad_style)] +mod c_backend { + use libc::{c_char, c_int}; + use std::mem; + + pub use libz_sys::deflate as mz_deflate; + pub use libz_sys::deflateEnd as mz_deflateEnd; + pub use libz_sys::deflateReset as mz_deflateReset; + pub use libz_sys::inflate as mz_inflate; + pub use libz_sys::inflateEnd as mz_inflateEnd; + pub use libz_sys::z_stream as mz_stream; + pub use libz_sys::*; + + pub use libz_sys::Z_BLOCK as MZ_BLOCK; + pub use libz_sys::Z_BUF_ERROR as MZ_BUF_ERROR; + pub use libz_sys::Z_DATA_ERROR as MZ_DATA_ERROR; + pub use libz_sys::Z_DEFAULT_STRATEGY as MZ_DEFAULT_STRATEGY; + pub use libz_sys::Z_DEFLATED as MZ_DEFLATED; + pub use libz_sys::Z_FINISH as MZ_FINISH; + pub use libz_sys::Z_FULL_FLUSH as MZ_FULL_FLUSH; + pub use libz_sys::Z_NEED_DICT as MZ_NEED_DICT; + pub use libz_sys::Z_NO_FLUSH as MZ_NO_FLUSH; + pub use libz_sys::Z_OK as MZ_OK; + pub use libz_sys::Z_PARTIAL_FLUSH as MZ_PARTIAL_FLUSH; + pub use libz_sys::Z_STREAM_END as MZ_STREAM_END; + pub use libz_sys::Z_STREAM_ERROR as MZ_STREAM_ERROR; + pub use libz_sys::Z_SYNC_FLUSH as MZ_SYNC_FLUSH; + pub type AllocSize = libz_sys::uInt; + + pub const MZ_DEFAULT_WINDOW_BITS: c_int = 15; + + const ZLIB_VERSION: &'static str = "1.2.8\0"; + + pub unsafe extern "C" fn mz_deflateInit2( + stream: *mut mz_stream, + level: c_int, + method: c_int, + window_bits: c_int, + mem_level: c_int, + strategy: c_int, + ) -> c_int { + libz_sys::deflateInit2_( + stream, + level, + method, + window_bits, + mem_level, + strategy, + ZLIB_VERSION.as_ptr() as *const c_char, + mem::size_of::<mz_stream>() as c_int, + ) + } + pub unsafe extern "C" fn mz_inflateInit2(stream: *mut mz_stream, window_bits: c_int) -> c_int { + libz_sys::inflateInit2_( + stream, + window_bits, + ZLIB_VERSION.as_ptr() as *const c_char, + mem::size_of::<mz_stream>() as c_int, + ) + } +} + +/// Cloudflare optimized Zlib specific +#[cfg(all(feature = "cloudflare_zlib", not(feature = "zlib-ng-compat")))] +#[allow(bad_style)] +mod c_backend { + use libc::{c_char, c_int}; + use std::mem; + + pub use cloudflare_zlib_sys::deflate as mz_deflate; + pub use cloudflare_zlib_sys::deflateEnd as mz_deflateEnd; + pub use cloudflare_zlib_sys::deflateReset as mz_deflateReset; + pub use cloudflare_zlib_sys::inflate as mz_inflate; + pub use cloudflare_zlib_sys::inflateEnd as mz_inflateEnd; + pub use cloudflare_zlib_sys::z_stream as mz_stream; + pub use cloudflare_zlib_sys::*; + + pub use cloudflare_zlib_sys::Z_BLOCK as MZ_BLOCK; + pub use cloudflare_zlib_sys::Z_BUF_ERROR as MZ_BUF_ERROR; + pub use cloudflare_zlib_sys::Z_DATA_ERROR as MZ_DATA_ERROR; + pub use cloudflare_zlib_sys::Z_DEFAULT_STRATEGY as MZ_DEFAULT_STRATEGY; + pub use cloudflare_zlib_sys::Z_DEFLATED as MZ_DEFLATED; + pub use cloudflare_zlib_sys::Z_FINISH as MZ_FINISH; + pub use cloudflare_zlib_sys::Z_FULL_FLUSH as MZ_FULL_FLUSH; + pub use cloudflare_zlib_sys::Z_NEED_DICT as MZ_NEED_DICT; + pub use cloudflare_zlib_sys::Z_NO_FLUSH as MZ_NO_FLUSH; + pub use cloudflare_zlib_sys::Z_OK as MZ_OK; + pub use cloudflare_zlib_sys::Z_PARTIAL_FLUSH as MZ_PARTIAL_FLUSH; + pub use cloudflare_zlib_sys::Z_STREAM_END as MZ_STREAM_END; + pub use cloudflare_zlib_sys::Z_STREAM_ERROR as MZ_STREAM_ERROR; + pub use cloudflare_zlib_sys::Z_SYNC_FLUSH as MZ_SYNC_FLUSH; + pub type AllocSize = cloudflare_zlib_sys::uInt; + + pub const MZ_DEFAULT_WINDOW_BITS: c_int = 15; + + const ZLIB_VERSION: &'static str = "1.2.8\0"; + + pub unsafe extern "C" fn mz_deflateInit2( + stream: *mut mz_stream, + level: c_int, + method: c_int, + window_bits: c_int, + mem_level: c_int, + strategy: c_int, + ) -> c_int { + cloudflare_zlib_sys::deflateInit2_( + stream, + level, + method, + window_bits, + mem_level, + strategy, + ZLIB_VERSION.as_ptr() as *const c_char, + mem::size_of::<mz_stream>() as c_int, + ) + } + pub unsafe extern "C" fn mz_inflateInit2(stream: *mut mz_stream, window_bits: c_int) -> c_int { + cloudflare_zlib_sys::inflateInit2_( + stream, + window_bits, + ZLIB_VERSION.as_ptr() as *const c_char, + mem::size_of::<mz_stream>() as c_int, + ) + } +} diff --git a/third_party/rust/flate2/src/ffi/mod.rs b/third_party/rust/flate2/src/ffi/mod.rs new file mode 100644 index 0000000000..4135e1143a --- /dev/null +++ b/third_party/rust/flate2/src/ffi/mod.rs @@ -0,0 +1,46 @@ +//! This module contains backend-specific code. + +use crate::mem::{CompressError, DecompressError, FlushCompress, FlushDecompress, Status}; +use crate::Compression; + +/// Traits specifying the interface of the backends. +/// +/// Sync + Send are added as a condition to ensure they are available +/// for the frontend. +pub trait Backend: Sync + Send { + fn total_in(&self) -> u64; + fn total_out(&self) -> u64; +} + +pub trait InflateBackend: Backend { + fn make(zlib_header: bool, window_bits: u8) -> Self; + fn decompress( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushDecompress, + ) -> Result<Status, DecompressError>; + fn reset(&mut self, zlib_header: bool); +} + +pub trait DeflateBackend: Backend { + fn make(level: Compression, zlib_header: bool, window_bits: u8) -> Self; + fn compress( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushCompress, + ) -> Result<Status, CompressError>; + fn reset(&mut self); +} + +// Default to Rust implementation unless explicitly opted in to a different backend. +cfg_if::cfg_if! { + if #[cfg(any(feature = "miniz-sys", feature = "any_zlib"))] { + mod c; + pub use self::c::*; + } else { + mod rust; + pub use self::rust::*; + } +} diff --git a/third_party/rust/flate2/src/ffi/rust.rs b/third_party/rust/flate2/src/ffi/rust.rs new file mode 100644 index 0000000000..2e62cef090 --- /dev/null +++ b/third_party/rust/flate2/src/ffi/rust.rs @@ -0,0 +1,183 @@ +//! Implementation for miniz_oxide rust backend. + +use std::convert::TryInto; +use std::fmt; + +use miniz_oxide::deflate::core::CompressorOxide; +use miniz_oxide::inflate::stream::InflateState; +pub use miniz_oxide::*; + +pub const MZ_NO_FLUSH: isize = MZFlush::None as isize; +pub const MZ_PARTIAL_FLUSH: isize = MZFlush::Partial as isize; +pub const MZ_SYNC_FLUSH: isize = MZFlush::Sync as isize; +pub const MZ_FULL_FLUSH: isize = MZFlush::Full as isize; +pub const MZ_FINISH: isize = MZFlush::Finish as isize; + +use super::*; +use crate::mem; + +fn format_from_bool(zlib_header: bool) -> DataFormat { + if zlib_header { + DataFormat::Zlib + } else { + DataFormat::Raw + } +} + +pub struct Inflate { + inner: Box<InflateState>, + total_in: u64, + total_out: u64, +} + +impl fmt::Debug for Inflate { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!( + f, + "miniz_oxide inflate internal state. total_in: {}, total_out: {}", + self.total_in, self.total_out, + ) + } +} + +impl InflateBackend for Inflate { + fn make(zlib_header: bool, window_bits: u8) -> Self { + assert!( + window_bits > 8 && window_bits < 16, + "window_bits must be within 9 ..= 15" + ); + + let format = format_from_bool(zlib_header); + + Inflate { + inner: InflateState::new_boxed(format), + total_in: 0, + total_out: 0, + } + } + + fn decompress( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushDecompress, + ) -> Result<Status, DecompressError> { + let flush = MZFlush::new(flush as i32).unwrap(); + + let res = inflate::stream::inflate(&mut self.inner, input, output, flush); + self.total_in += res.bytes_consumed as u64; + self.total_out += res.bytes_written as u64; + + match res.status { + Ok(status) => match status { + MZStatus::Ok => Ok(Status::Ok), + MZStatus::StreamEnd => Ok(Status::StreamEnd), + MZStatus::NeedDict => { + mem::decompress_need_dict(self.inner.decompressor().adler32().unwrap_or(0)) + } + }, + Err(status) => match status { + MZError::Buf => Ok(Status::BufError), + _ => mem::decompress_failed(), + }, + } + } + + fn reset(&mut self, zlib_header: bool) { + self.inner.reset(format_from_bool(zlib_header)); + self.total_in = 0; + self.total_out = 0; + } +} + +impl Backend for Inflate { + #[inline] + fn total_in(&self) -> u64 { + self.total_in + } + + #[inline] + fn total_out(&self) -> u64 { + self.total_out + } +} + +pub struct Deflate { + inner: Box<CompressorOxide>, + total_in: u64, + total_out: u64, +} + +impl fmt::Debug for Deflate { + fn fmt(&self, f: &mut fmt::Formatter) -> Result<(), fmt::Error> { + write!( + f, + "miniz_oxide deflate internal state. total_in: {}, total_out: {}", + self.total_in, self.total_out, + ) + } +} + +impl DeflateBackend for Deflate { + fn make(level: Compression, zlib_header: bool, window_bits: u8) -> Self { + assert!( + window_bits > 8 && window_bits < 16, + "window_bits must be within 9 ..= 15" + ); + + // Check in case the integer value changes at some point. + debug_assert!(level.level() <= 10); + + let mut inner: Box<CompressorOxide> = Box::default(); + let format = format_from_bool(zlib_header); + inner.set_format_and_level(format, level.level().try_into().unwrap_or(1)); + + Deflate { + inner, + total_in: 0, + total_out: 0, + } + } + + fn compress( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushCompress, + ) -> Result<Status, CompressError> { + let flush = MZFlush::new(flush as i32).unwrap(); + let res = deflate::stream::deflate(&mut self.inner, input, output, flush); + self.total_in += res.bytes_consumed as u64; + self.total_out += res.bytes_written as u64; + + match res.status { + Ok(status) => match status { + MZStatus::Ok => Ok(Status::Ok), + MZStatus::StreamEnd => Ok(Status::StreamEnd), + MZStatus::NeedDict => Err(CompressError(())), + }, + Err(status) => match status { + MZError::Buf => Ok(Status::BufError), + _ => Err(CompressError(())), + }, + } + } + + fn reset(&mut self) { + self.total_in = 0; + self.total_out = 0; + self.inner.reset(); + } +} + +impl Backend for Deflate { + #[inline] + fn total_in(&self) -> u64 { + self.total_in + } + + #[inline] + fn total_out(&self) -> u64 { + self.total_out + } +} diff --git a/third_party/rust/flate2/src/gz/bufread.rs b/third_party/rust/flate2/src/gz/bufread.rs new file mode 100644 index 0000000000..b66ab72f82 --- /dev/null +++ b/third_party/rust/flate2/src/gz/bufread.rs @@ -0,0 +1,641 @@ +use std::cmp; +use std::io; +use std::io::prelude::*; +use std::mem; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use super::{GzBuilder, GzHeader}; +use super::{FCOMMENT, FEXTRA, FHCRC, FNAME}; +use crate::crc::CrcReader; +use crate::deflate; +use crate::Compression; + +fn copy(into: &mut [u8], from: &[u8], pos: &mut usize) -> usize { + let min = cmp::min(into.len(), from.len() - *pos); + for (slot, val) in into.iter_mut().zip(from[*pos..*pos + min].iter()) { + *slot = *val; + } + *pos += min; + return min; +} + +pub(crate) fn corrupt() -> io::Error { + io::Error::new( + io::ErrorKind::InvalidInput, + "corrupt gzip stream does not have a matching checksum", + ) +} + +fn bad_header() -> io::Error { + io::Error::new(io::ErrorKind::InvalidInput, "invalid gzip header") +} + +fn read_le_u16<R: Read>(r: &mut R) -> io::Result<u16> { + let mut b = [0; 2]; + r.read_exact(&mut b)?; + Ok((b[0] as u16) | ((b[1] as u16) << 8)) +} + +pub(crate) fn read_gz_header<R: Read>(r: &mut R) -> io::Result<GzHeader> { + let mut crc_reader = CrcReader::new(r); + let mut header = [0; 10]; + crc_reader.read_exact(&mut header)?; + + let id1 = header[0]; + let id2 = header[1]; + if id1 != 0x1f || id2 != 0x8b { + return Err(bad_header()); + } + let cm = header[2]; + if cm != 8 { + return Err(bad_header()); + } + + let flg = header[3]; + let mtime = ((header[4] as u32) << 0) + | ((header[5] as u32) << 8) + | ((header[6] as u32) << 16) + | ((header[7] as u32) << 24); + let _xfl = header[8]; + let os = header[9]; + + let extra = if flg & FEXTRA != 0 { + let xlen = read_le_u16(&mut crc_reader)?; + let mut extra = vec![0; xlen as usize]; + crc_reader.read_exact(&mut extra)?; + Some(extra) + } else { + None + }; + let filename = if flg & FNAME != 0 { + // wow this is slow + let mut b = Vec::new(); + for byte in crc_reader.by_ref().bytes() { + let byte = byte?; + if byte == 0 { + break; + } + b.push(byte); + } + Some(b) + } else { + None + }; + let comment = if flg & FCOMMENT != 0 { + // wow this is slow + let mut b = Vec::new(); + for byte in crc_reader.by_ref().bytes() { + let byte = byte?; + if byte == 0 { + break; + } + b.push(byte); + } + Some(b) + } else { + None + }; + + if flg & FHCRC != 0 { + let calced_crc = crc_reader.crc().sum() as u16; + let stored_crc = read_le_u16(&mut crc_reader)?; + if calced_crc != stored_crc { + return Err(corrupt()); + } + } + + Ok(GzHeader { + extra: extra, + filename: filename, + comment: comment, + operating_system: os, + mtime: mtime, + }) +} + +/// A gzip streaming encoder +/// +/// This structure exposes a [`BufRead`] interface that will read uncompressed data +/// from the underlying reader and expose the compressed version as a [`BufRead`] +/// interface. +/// +/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// use flate2::Compression; +/// use flate2::bufread::GzEncoder; +/// use std::fs::File; +/// use std::io::BufReader; +/// +/// // Opens sample file, compresses the contents and returns a Vector or error +/// // File wrapped in a BufReader implements BufRead +/// +/// fn open_hello_world() -> io::Result<Vec<u8>> { +/// let f = File::open("examples/hello_world.txt")?; +/// let b = BufReader::new(f); +/// let mut gz = GzEncoder::new(b, Compression::fast()); +/// let mut buffer = Vec::new(); +/// gz.read_to_end(&mut buffer)?; +/// Ok(buffer) +/// } +/// ``` +#[derive(Debug)] +pub struct GzEncoder<R> { + inner: deflate::bufread::DeflateEncoder<CrcReader<R>>, + header: Vec<u8>, + pos: usize, + eof: bool, +} + +pub fn gz_encoder<R: BufRead>(header: Vec<u8>, r: R, lvl: Compression) -> GzEncoder<R> { + let crc = CrcReader::new(r); + GzEncoder { + inner: deflate::bufread::DeflateEncoder::new(crc, lvl), + header: header, + pos: 0, + eof: false, + } +} + +impl<R: BufRead> GzEncoder<R> { + /// Creates a new encoder which will use the given compression level. + /// + /// The encoder is not configured specially for the emitted header. For + /// header configuration, see the `GzBuilder` type. + /// + /// The data read from the stream `r` will be compressed and available + /// through the returned reader. + pub fn new(r: R, level: Compression) -> GzEncoder<R> { + GzBuilder::new().buf_read(r, level) + } + + fn read_footer(&mut self, into: &mut [u8]) -> io::Result<usize> { + if self.pos == 8 { + return Ok(0); + } + let crc = self.inner.get_ref().crc(); + let ref arr = [ + (crc.sum() >> 0) as u8, + (crc.sum() >> 8) as u8, + (crc.sum() >> 16) as u8, + (crc.sum() >> 24) as u8, + (crc.amount() >> 0) as u8, + (crc.amount() >> 8) as u8, + (crc.amount() >> 16) as u8, + (crc.amount() >> 24) as u8, + ]; + Ok(copy(into, arr, &mut self.pos)) + } +} + +impl<R> GzEncoder<R> { + /// Acquires a reference to the underlying reader. + pub fn get_ref(&self) -> &R { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying reader. + /// + /// Note that mutation of the reader may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.inner.get_mut().get_mut() + } + + /// Returns the underlying stream, consuming this encoder + pub fn into_inner(self) -> R { + self.inner.into_inner().into_inner() + } +} + +#[inline] +fn finish(buf: &[u8; 8]) -> (u32, u32) { + let crc = ((buf[0] as u32) << 0) + | ((buf[1] as u32) << 8) + | ((buf[2] as u32) << 16) + | ((buf[3] as u32) << 24); + let amt = ((buf[4] as u32) << 0) + | ((buf[5] as u32) << 8) + | ((buf[6] as u32) << 16) + | ((buf[7] as u32) << 24); + (crc, amt) +} + +impl<R: BufRead> Read for GzEncoder<R> { + fn read(&mut self, mut into: &mut [u8]) -> io::Result<usize> { + let mut amt = 0; + if self.eof { + return self.read_footer(into); + } else if self.pos < self.header.len() { + amt += copy(into, &self.header, &mut self.pos); + if amt == into.len() { + return Ok(amt); + } + let tmp = into; + into = &mut tmp[amt..]; + } + match self.inner.read(into)? { + 0 => { + self.eof = true; + self.pos = 0; + self.read_footer(into) + } + n => Ok(amt + n), + } + } +} + +impl<R: BufRead + Write> Write for GzEncoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +/// A gzip streaming decoder +/// +/// This structure consumes a [`BufRead`] interface, reading compressed data +/// from the underlying reader, and emitting uncompressed data. +/// +/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::GzEncoder; +/// use flate2::bufread::GzDecoder; +/// +/// # fn main() { +/// # let mut e = GzEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_reader(bytes).unwrap()); +/// # } +/// # +/// // Uncompresses a Gz Encoded vector of bytes and returns a string or error +/// // Here &[u8] implements BufRead +/// +/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut gz = GzDecoder::new(&bytes[..]); +/// let mut s = String::new(); +/// gz.read_to_string(&mut s)?; +/// Ok(s) +/// } +/// ``` +#[derive(Debug)] +pub struct GzDecoder<R> { + inner: GzState, + header: Option<GzHeader>, + reader: CrcReader<deflate::bufread::DeflateDecoder<R>>, + multi: bool, +} + +#[derive(Debug)] +enum GzState { + Header(Vec<u8>), + Body, + Finished(usize, [u8; 8]), + Err(io::Error), + End, +} + +/// A small adapter which reads data originally from `buf` and then reads all +/// further data from `reader`. This will also buffer all data read from +/// `reader` into `buf` for reuse on a further call. +struct Buffer<'a, T: 'a> { + buf: &'a mut Vec<u8>, + buf_cur: usize, + buf_max: usize, + reader: &'a mut T, +} + +impl<'a, T> Buffer<'a, T> { + fn new(buf: &'a mut Vec<u8>, reader: &'a mut T) -> Buffer<'a, T> { + Buffer { + reader, + buf_cur: 0, + buf_max: buf.len(), + buf, + } + } +} + +impl<'a, T: Read> Read for Buffer<'a, T> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + if self.buf_cur == self.buf_max { + let len = self.reader.read(buf)?; + self.buf.extend_from_slice(&buf[..len]); + Ok(len) + } else { + let len = (&self.buf[self.buf_cur..self.buf_max]).read(buf)?; + self.buf_cur += len; + Ok(len) + } + } +} + +impl<R: BufRead> GzDecoder<R> { + /// Creates a new decoder from the given reader, immediately parsing the + /// gzip header. + pub fn new(mut r: R) -> GzDecoder<R> { + let mut buf = Vec::with_capacity(10); // minimum header length + let mut header = None; + + let result = { + let mut reader = Buffer::new(&mut buf, &mut r); + read_gz_header(&mut reader) + }; + + let state = match result { + Ok(hdr) => { + header = Some(hdr); + GzState::Body + } + Err(ref err) if io::ErrorKind::WouldBlock == err.kind() => GzState::Header(buf), + Err(err) => GzState::Err(err), + }; + + GzDecoder { + inner: state, + reader: CrcReader::new(deflate::bufread::DeflateDecoder::new(r)), + multi: false, + header, + } + } + + fn multi(mut self, flag: bool) -> GzDecoder<R> { + self.multi = flag; + self + } +} + +impl<R> GzDecoder<R> { + /// Returns the header associated with this stream, if it was valid + pub fn header(&self) -> Option<&GzHeader> { + self.header.as_ref() + } + + /// Acquires a reference to the underlying reader. + pub fn get_ref(&self) -> &R { + self.reader.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying stream. + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.reader.get_mut().get_mut() + } + + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.reader.into_inner().into_inner() + } +} + +impl<R: BufRead> Read for GzDecoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + let GzDecoder { + inner, + header, + reader, + multi, + } = self; + + loop { + *inner = match mem::replace(inner, GzState::End) { + GzState::Header(mut buf) => { + let result = { + let mut reader = Buffer::new(&mut buf, reader.get_mut().get_mut()); + read_gz_header(&mut reader) + }; + let hdr = result.map_err(|err| { + if io::ErrorKind::WouldBlock == err.kind() { + *inner = GzState::Header(buf); + } + + err + })?; + *header = Some(hdr); + GzState::Body + } + GzState::Body => { + if into.is_empty() { + *inner = GzState::Body; + return Ok(0); + } + + let n = reader.read(into).map_err(|err| { + if io::ErrorKind::WouldBlock == err.kind() { + *inner = GzState::Body; + } + + err + })?; + + match n { + 0 => GzState::Finished(0, [0; 8]), + n => { + *inner = GzState::Body; + return Ok(n); + } + } + } + GzState::Finished(pos, mut buf) => { + if pos < buf.len() { + let n = reader + .get_mut() + .get_mut() + .read(&mut buf[pos..]) + .and_then(|n| { + if n == 0 { + Err(io::ErrorKind::UnexpectedEof.into()) + } else { + Ok(n) + } + }) + .map_err(|err| { + if io::ErrorKind::WouldBlock == err.kind() { + *inner = GzState::Finished(pos, buf); + } + + err + })?; + + GzState::Finished(pos + n, buf) + } else { + let (crc, amt) = finish(&buf); + + if crc != reader.crc().sum() || amt != reader.crc().amount() { + return Err(corrupt()); + } else if *multi { + let is_eof = reader + .get_mut() + .get_mut() + .fill_buf() + .map(|buf| buf.is_empty()) + .map_err(|err| { + if io::ErrorKind::WouldBlock == err.kind() { + *inner = GzState::Finished(pos, buf); + } + + err + })?; + + if is_eof { + GzState::End + } else { + reader.reset(); + reader.get_mut().reset_data(); + header.take(); + GzState::Header(Vec::with_capacity(10)) + } + } else { + GzState::End + } + } + } + GzState::Err(err) => return Err(err), + GzState::End => return Ok(0), + }; + } + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + BufRead> AsyncRead for GzDecoder<R> {} + +impl<R: BufRead + Write> Write for GzDecoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + BufRead> AsyncWrite for GzDecoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} + +/// A gzip streaming decoder that decodes all members of a multistream +/// +/// A gzip member consists of a header, compressed data and a trailer. The [gzip +/// specification](https://tools.ietf.org/html/rfc1952), however, allows multiple +/// gzip members to be joined in a single stream. `MultiGzDecoder` will +/// decode all consecutive members while `GzDecoder` will only decompress +/// the first gzip member. The multistream format is commonly used in +/// bioinformatics, for example when using the BGZF compressed data. +/// +/// This structure exposes a [`BufRead`] interface that will consume all gzip members +/// from the underlying reader and emit uncompressed data. +/// +/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::GzEncoder; +/// use flate2::bufread::MultiGzDecoder; +/// +/// # fn main() { +/// # let mut e = GzEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_reader(bytes).unwrap()); +/// # } +/// # +/// // Uncompresses a Gz Encoded vector of bytes and returns a string or error +/// // Here &[u8] implements BufRead +/// +/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut gz = MultiGzDecoder::new(&bytes[..]); +/// let mut s = String::new(); +/// gz.read_to_string(&mut s)?; +/// Ok(s) +/// } +/// ``` +#[derive(Debug)] +pub struct MultiGzDecoder<R>(GzDecoder<R>); + +impl<R: BufRead> MultiGzDecoder<R> { + /// Creates a new decoder from the given reader, immediately parsing the + /// (first) gzip header. If the gzip stream contains multiple members all will + /// be decoded. + pub fn new(r: R) -> MultiGzDecoder<R> { + MultiGzDecoder(GzDecoder::new(r).multi(true)) + } +} + +impl<R> MultiGzDecoder<R> { + /// Returns the current header associated with this stream, if it's valid + pub fn header(&self) -> Option<&GzHeader> { + self.0.header() + } + + /// Acquires a reference to the underlying reader. + pub fn get_ref(&self) -> &R { + self.0.get_ref() + } + + /// Acquires a mutable reference to the underlying stream. + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.0.get_mut() + } + + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.0.into_inner() + } +} + +impl<R: BufRead> Read for MultiGzDecoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + self.0.read(into) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + BufRead> AsyncRead for MultiGzDecoder<R> {} + +impl<R: BufRead + Write> Write for MultiGzDecoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + BufRead> AsyncWrite for MultiGzDecoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} diff --git a/third_party/rust/flate2/src/gz/mod.rs b/third_party/rust/flate2/src/gz/mod.rs new file mode 100644 index 0000000000..3108035749 --- /dev/null +++ b/third_party/rust/flate2/src/gz/mod.rs @@ -0,0 +1,359 @@ +use std::ffi::CString; +use std::io::prelude::*; +use std::time; + +use crate::bufreader::BufReader; +use crate::Compression; + +pub static FHCRC: u8 = 1 << 1; +pub static FEXTRA: u8 = 1 << 2; +pub static FNAME: u8 = 1 << 3; +pub static FCOMMENT: u8 = 1 << 4; + +pub mod bufread; +pub mod read; +pub mod write; + +/// A structure representing the header of a gzip stream. +/// +/// The header can contain metadata about the file that was compressed, if +/// present. +#[derive(PartialEq, Clone, Debug, Default)] +pub struct GzHeader { + extra: Option<Vec<u8>>, + filename: Option<Vec<u8>>, + comment: Option<Vec<u8>>, + operating_system: u8, + mtime: u32, +} + +impl GzHeader { + /// Returns the `filename` field of this gzip stream's header, if present. + pub fn filename(&self) -> Option<&[u8]> { + self.filename.as_ref().map(|s| &s[..]) + } + + /// Returns the `extra` field of this gzip stream's header, if present. + pub fn extra(&self) -> Option<&[u8]> { + self.extra.as_ref().map(|s| &s[..]) + } + + /// Returns the `comment` field of this gzip stream's header, if present. + pub fn comment(&self) -> Option<&[u8]> { + self.comment.as_ref().map(|s| &s[..]) + } + + /// Returns the `operating_system` field of this gzip stream's header. + /// + /// There are predefined values for various operating systems. + /// 255 means that the value is unknown. + pub fn operating_system(&self) -> u8 { + self.operating_system + } + + /// This gives the most recent modification time of the original file being compressed. + /// + /// The time is in Unix format, i.e., seconds since 00:00:00 GMT, Jan. 1, 1970. + /// (Note that this may cause problems for MS-DOS and other systems that use local + /// rather than Universal time.) If the compressed data did not come from a file, + /// `mtime` is set to the time at which compression started. + /// `mtime` = 0 means no time stamp is available. + /// + /// The usage of `mtime` is discouraged because of Year 2038 problem. + pub fn mtime(&self) -> u32 { + self.mtime + } + + /// Returns the most recent modification time represented by a date-time type. + /// Returns `None` if the value of the underlying counter is 0, + /// indicating no time stamp is available. + /// + /// + /// The time is measured as seconds since 00:00:00 GMT, Jan. 1 1970. + /// See [`mtime`](#method.mtime) for more detail. + pub fn mtime_as_datetime(&self) -> Option<time::SystemTime> { + if self.mtime == 0 { + None + } else { + let duration = time::Duration::new(u64::from(self.mtime), 0); + let datetime = time::UNIX_EPOCH + duration; + Some(datetime) + } + } +} + +/// A builder structure to create a new gzip Encoder. +/// +/// This structure controls header configuration options such as the filename. +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// # use std::io; +/// use std::fs::File; +/// use flate2::GzBuilder; +/// use flate2::Compression; +/// +/// // GzBuilder opens a file and writes a sample string using GzBuilder pattern +/// +/// # fn sample_builder() -> Result<(), io::Error> { +/// let f = File::create("examples/hello_world.gz")?; +/// let mut gz = GzBuilder::new() +/// .filename("hello_world.txt") +/// .comment("test file, please delete") +/// .write(f, Compression::default()); +/// gz.write_all(b"hello world")?; +/// gz.finish()?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug)] +pub struct GzBuilder { + extra: Option<Vec<u8>>, + filename: Option<CString>, + comment: Option<CString>, + operating_system: Option<u8>, + mtime: u32, +} + +impl GzBuilder { + /// Create a new blank builder with no header by default. + pub fn new() -> GzBuilder { + GzBuilder { + extra: None, + filename: None, + comment: None, + operating_system: None, + mtime: 0, + } + } + + /// Configure the `mtime` field in the gzip header. + pub fn mtime(mut self, mtime: u32) -> GzBuilder { + self.mtime = mtime; + self + } + + /// Configure the `operating_system` field in the gzip header. + pub fn operating_system(mut self, os: u8) -> GzBuilder { + self.operating_system = Some(os); + self + } + + /// Configure the `extra` field in the gzip header. + pub fn extra<T: Into<Vec<u8>>>(mut self, extra: T) -> GzBuilder { + self.extra = Some(extra.into()); + self + } + + /// Configure the `filename` field in the gzip header. + /// + /// # Panics + /// + /// Panics if the `filename` slice contains a zero. + pub fn filename<T: Into<Vec<u8>>>(mut self, filename: T) -> GzBuilder { + self.filename = Some(CString::new(filename.into()).unwrap()); + self + } + + /// Configure the `comment` field in the gzip header. + /// + /// # Panics + /// + /// Panics if the `comment` slice contains a zero. + pub fn comment<T: Into<Vec<u8>>>(mut self, comment: T) -> GzBuilder { + self.comment = Some(CString::new(comment.into()).unwrap()); + self + } + + /// Consume this builder, creating a writer encoder in the process. + /// + /// The data written to the returned encoder will be compressed and then + /// written out to the supplied parameter `w`. + pub fn write<W: Write>(self, w: W, lvl: Compression) -> write::GzEncoder<W> { + write::gz_encoder(self.into_header(lvl), w, lvl) + } + + /// Consume this builder, creating a reader encoder in the process. + /// + /// Data read from the returned encoder will be the compressed version of + /// the data read from the given reader. + pub fn read<R: Read>(self, r: R, lvl: Compression) -> read::GzEncoder<R> { + read::gz_encoder(self.buf_read(BufReader::new(r), lvl)) + } + + /// Consume this builder, creating a reader encoder in the process. + /// + /// Data read from the returned encoder will be the compressed version of + /// the data read from the given reader. + pub fn buf_read<R>(self, r: R, lvl: Compression) -> bufread::GzEncoder<R> + where + R: BufRead, + { + bufread::gz_encoder(self.into_header(lvl), r, lvl) + } + + fn into_header(self, lvl: Compression) -> Vec<u8> { + let GzBuilder { + extra, + filename, + comment, + operating_system, + mtime, + } = self; + let mut flg = 0; + let mut header = vec![0u8; 10]; + match extra { + Some(v) => { + flg |= FEXTRA; + header.push((v.len() >> 0) as u8); + header.push((v.len() >> 8) as u8); + header.extend(v); + } + None => {} + } + match filename { + Some(filename) => { + flg |= FNAME; + header.extend(filename.as_bytes_with_nul().iter().map(|x| *x)); + } + None => {} + } + match comment { + Some(comment) => { + flg |= FCOMMENT; + header.extend(comment.as_bytes_with_nul().iter().map(|x| *x)); + } + None => {} + } + header[0] = 0x1f; + header[1] = 0x8b; + header[2] = 8; + header[3] = flg; + header[4] = (mtime >> 0) as u8; + header[5] = (mtime >> 8) as u8; + header[6] = (mtime >> 16) as u8; + header[7] = (mtime >> 24) as u8; + header[8] = if lvl.0 >= Compression::best().0 { + 2 + } else if lvl.0 <= Compression::fast().0 { + 4 + } else { + 0 + }; + + // Typically this byte indicates what OS the gz stream was created on, + // but in an effort to have cross-platform reproducible streams just + // default this value to 255. I'm not sure that if we "correctly" set + // this it'd do anything anyway... + header[9] = operating_system.unwrap_or(255); + return header; + } +} + +#[cfg(test)] +mod tests { + use std::io::prelude::*; + + use super::{read, write, GzBuilder}; + use crate::Compression; + use rand::{thread_rng, Rng}; + + #[test] + fn roundtrip() { + let mut e = write::GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"foo bar baz").unwrap(); + let inner = e.finish().unwrap(); + let mut d = read::GzDecoder::new(&inner[..]); + let mut s = String::new(); + d.read_to_string(&mut s).unwrap(); + assert_eq!(s, "foo bar baz"); + } + + #[test] + fn roundtrip_zero() { + let e = write::GzEncoder::new(Vec::new(), Compression::default()); + let inner = e.finish().unwrap(); + let mut d = read::GzDecoder::new(&inner[..]); + let mut s = String::new(); + d.read_to_string(&mut s).unwrap(); + assert_eq!(s, ""); + } + + #[test] + fn roundtrip_big() { + let mut real = Vec::new(); + let mut w = write::GzEncoder::new(Vec::new(), Compression::default()); + let v = crate::random_bytes().take(1024).collect::<Vec<_>>(); + for _ in 0..200 { + let to_write = &v[..thread_rng().gen_range(0, v.len())]; + real.extend(to_write.iter().map(|x| *x)); + w.write_all(to_write).unwrap(); + } + let result = w.finish().unwrap(); + let mut r = read::GzDecoder::new(&result[..]); + let mut v = Vec::new(); + r.read_to_end(&mut v).unwrap(); + assert!(v == real); + } + + #[test] + fn roundtrip_big2() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let mut r = read::GzDecoder::new(read::GzEncoder::new(&v[..], Compression::default())); + let mut res = Vec::new(); + r.read_to_end(&mut res).unwrap(); + assert!(res == v); + } + + #[test] + fn fields() { + let r = vec![0, 2, 4, 6]; + let e = GzBuilder::new() + .filename("foo.rs") + .comment("bar") + .extra(vec![0, 1, 2, 3]) + .read(&r[..], Compression::default()); + let mut d = read::GzDecoder::new(e); + assert_eq!(d.header().unwrap().filename(), Some(&b"foo.rs"[..])); + assert_eq!(d.header().unwrap().comment(), Some(&b"bar"[..])); + assert_eq!(d.header().unwrap().extra(), Some(&b"\x00\x01\x02\x03"[..])); + let mut res = Vec::new(); + d.read_to_end(&mut res).unwrap(); + assert_eq!(res, vec![0, 2, 4, 6]); + } + + #[test] + fn keep_reading_after_end() { + let mut e = write::GzEncoder::new(Vec::new(), Compression::default()); + e.write_all(b"foo bar baz").unwrap(); + let inner = e.finish().unwrap(); + let mut d = read::GzDecoder::new(&inner[..]); + let mut s = String::new(); + d.read_to_string(&mut s).unwrap(); + assert_eq!(s, "foo bar baz"); + d.read_to_string(&mut s).unwrap(); + assert_eq!(s, "foo bar baz"); + } + + #[test] + fn qc_reader() { + ::quickcheck::quickcheck(test as fn(_) -> _); + + fn test(v: Vec<u8>) -> bool { + let r = read::GzEncoder::new(&v[..], Compression::default()); + let mut r = read::GzDecoder::new(r); + let mut v2 = Vec::new(); + r.read_to_end(&mut v2).unwrap(); + v == v2 + } + } + + #[test] + fn flush_after_write() { + let mut f = write::GzEncoder::new(Vec::new(), Compression::default()); + write!(f, "Hello world").unwrap(); + f.flush().unwrap(); + } +} diff --git a/third_party/rust/flate2/src/gz/read.rs b/third_party/rust/flate2/src/gz/read.rs new file mode 100644 index 0000000000..25f2741dc8 --- /dev/null +++ b/third_party/rust/flate2/src/gz/read.rs @@ -0,0 +1,303 @@ +use std::io; +use std::io::prelude::*; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use super::bufread; +use super::{GzBuilder, GzHeader}; +use crate::bufreader::BufReader; +use crate::Compression; + +/// A gzip streaming encoder +/// +/// This structure exposes a [`Read`] interface that will read uncompressed data +/// from the underlying reader and expose the compressed version as a [`Read`] +/// interface. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// use flate2::Compression; +/// use flate2::read::GzEncoder; +/// +/// // Return a vector containing the GZ compressed version of hello world +/// +/// fn gzencode_hello_world() -> io::Result<Vec<u8>> { +/// let mut ret_vec = [0;100]; +/// let bytestring = b"hello world"; +/// let mut gz = GzEncoder::new(&bytestring[..], Compression::fast()); +/// let count = gz.read(&mut ret_vec)?; +/// Ok(ret_vec[0..count].to_vec()) +/// } +/// ``` +#[derive(Debug)] +pub struct GzEncoder<R> { + inner: bufread::GzEncoder<BufReader<R>>, +} + +pub fn gz_encoder<R: Read>(inner: bufread::GzEncoder<BufReader<R>>) -> GzEncoder<R> { + GzEncoder { inner: inner } +} + +impl<R: Read> GzEncoder<R> { + /// Creates a new encoder which will use the given compression level. + /// + /// The encoder is not configured specially for the emitted header. For + /// header configuration, see the `GzBuilder` type. + /// + /// The data read from the stream `r` will be compressed and available + /// through the returned reader. + pub fn new(r: R, level: Compression) -> GzEncoder<R> { + GzBuilder::new().read(r, level) + } +} + +impl<R> GzEncoder<R> { + /// Acquires a reference to the underlying reader. + pub fn get_ref(&self) -> &R { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying reader. + /// + /// Note that mutation of the reader may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.inner.get_mut().get_mut() + } + + /// Returns the underlying stream, consuming this encoder + pub fn into_inner(self) -> R { + self.inner.into_inner().into_inner() + } +} + +impl<R: Read> Read for GzEncoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + self.inner.read(into) + } +} + +impl<R: Read + Write> Write for GzEncoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +/// A gzip streaming decoder +/// +/// This structure exposes a [`Read`] interface that will consume compressed +/// data from the underlying reader and emit uncompressed data. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// +/// # Examples +/// +/// ``` +/// +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::GzEncoder; +/// use flate2::read::GzDecoder; +/// +/// # fn main() { +/// # let mut e = GzEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_reader(bytes).unwrap()); +/// # } +/// # +/// // Uncompresses a Gz Encoded vector of bytes and returns a string or error +/// // Here &[u8] implements Read +/// +/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut gz = GzDecoder::new(&bytes[..]); +/// let mut s = String::new(); +/// gz.read_to_string(&mut s)?; +/// Ok(s) +/// } +/// ``` +#[derive(Debug)] +pub struct GzDecoder<R> { + inner: bufread::GzDecoder<BufReader<R>>, +} + +impl<R: Read> GzDecoder<R> { + /// Creates a new decoder from the given reader, immediately parsing the + /// gzip header. + pub fn new(r: R) -> GzDecoder<R> { + GzDecoder { + inner: bufread::GzDecoder::new(BufReader::new(r)), + } + } +} + +impl<R> GzDecoder<R> { + /// Returns the header associated with this stream, if it was valid. + pub fn header(&self) -> Option<&GzHeader> { + self.inner.header() + } + + /// Acquires a reference to the underlying reader. + pub fn get_ref(&self) -> &R { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying stream. + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.inner.get_mut().get_mut() + } + + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.inner.into_inner().into_inner() + } +} + +impl<R: Read> Read for GzDecoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + self.inner.read(into) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead> AsyncRead for GzDecoder<R> {} + +impl<R: Read + Write> Write for GzDecoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + AsyncRead> AsyncWrite for GzDecoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} + +/// A gzip streaming decoder that decodes all members of a multistream +/// +/// A gzip member consists of a header, compressed data and a trailer. The [gzip +/// specification](https://tools.ietf.org/html/rfc1952), however, allows multiple +/// gzip members to be joined in a single stream. `MultiGzDecoder` will +/// decode all consecutive members while `GzDecoder` will only decompress the +/// first gzip member. The multistream format is commonly used in bioinformatics, +/// for example when using the BGZF compressed data. +/// +/// This structure exposes a [`Read`] interface that will consume all gzip members +/// from the underlying reader and emit uncompressed data. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::GzEncoder; +/// use flate2::read::MultiGzDecoder; +/// +/// # fn main() { +/// # let mut e = GzEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_reader(bytes).unwrap()); +/// # } +/// # +/// // Uncompresses a Gz Encoded vector of bytes and returns a string or error +/// // Here &[u8] implements Read +/// +/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut gz = MultiGzDecoder::new(&bytes[..]); +/// let mut s = String::new(); +/// gz.read_to_string(&mut s)?; +/// Ok(s) +/// } +/// ``` +#[derive(Debug)] +pub struct MultiGzDecoder<R> { + inner: bufread::MultiGzDecoder<BufReader<R>>, +} + +impl<R: Read> MultiGzDecoder<R> { + /// Creates a new decoder from the given reader, immediately parsing the + /// (first) gzip header. If the gzip stream contains multiple members all will + /// be decoded. + pub fn new(r: R) -> MultiGzDecoder<R> { + MultiGzDecoder { + inner: bufread::MultiGzDecoder::new(BufReader::new(r)), + } + } +} + +impl<R> MultiGzDecoder<R> { + /// Returns the current header associated with this stream, if it's valid. + pub fn header(&self) -> Option<&GzHeader> { + self.inner.header() + } + + /// Acquires a reference to the underlying reader. + pub fn get_ref(&self) -> &R { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying stream. + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.inner.get_mut().get_mut() + } + + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.inner.into_inner().into_inner() + } +} + +impl<R: Read> Read for MultiGzDecoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + self.inner.read(into) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead> AsyncRead for MultiGzDecoder<R> {} + +impl<R: Read + Write> Write for MultiGzDecoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + AsyncRead> AsyncWrite for MultiGzDecoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} diff --git a/third_party/rust/flate2/src/gz/write.rs b/third_party/rust/flate2/src/gz/write.rs new file mode 100644 index 0000000000..4e275937bf --- /dev/null +++ b/third_party/rust/flate2/src/gz/write.rs @@ -0,0 +1,479 @@ +use std::cmp; +use std::io; +use std::io::prelude::*; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use super::bufread::{corrupt, read_gz_header}; +use super::{GzBuilder, GzHeader}; +use crate::crc::{Crc, CrcWriter}; +use crate::zio; +use crate::{Compress, Compression, Decompress, Status}; + +/// A gzip streaming encoder +/// +/// This structure exposes a [`Write`] interface that will emit compressed data +/// to the underlying writer `W`. +/// +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use flate2::Compression; +/// use flate2::write::GzEncoder; +/// +/// // Vec<u8> implements Write to print the compressed bytes of sample string +/// # fn main() { +/// +/// let mut e = GzEncoder::new(Vec::new(), Compression::default()); +/// e.write_all(b"Hello World").unwrap(); +/// println!("{:?}", e.finish().unwrap()); +/// # } +/// ``` +#[derive(Debug)] +pub struct GzEncoder<W: Write> { + inner: zio::Writer<W, Compress>, + crc: Crc, + crc_bytes_written: usize, + header: Vec<u8>, +} + +pub fn gz_encoder<W: Write>(header: Vec<u8>, w: W, lvl: Compression) -> GzEncoder<W> { + GzEncoder { + inner: zio::Writer::new(w, Compress::new(lvl, false)), + crc: Crc::new(), + header: header, + crc_bytes_written: 0, + } +} + +impl<W: Write> GzEncoder<W> { + /// Creates a new encoder which will use the given compression level. + /// + /// The encoder is not configured specially for the emitted header. For + /// header configuration, see the `GzBuilder` type. + /// + /// The data written to the returned encoder will be compressed and then + /// written to the stream `w`. + pub fn new(w: W, level: Compression) -> GzEncoder<W> { + GzBuilder::new().write(w, level) + } + + /// Acquires a reference to the underlying writer. + pub fn get_ref(&self) -> &W { + self.inner.get_ref() + } + + /// Acquires a mutable reference to the underlying writer. + /// + /// Note that mutation of the writer may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut W { + self.inner.get_mut() + } + + /// Attempt to finish this output stream, writing out final chunks of data. + /// + /// Note that this function can only be used once data has finished being + /// written to the output stream. After this function is called then further + /// calls to `write` may result in a panic. + /// + /// # Panics + /// + /// Attempts to write data to this stream may result in a panic after this + /// function is called. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn try_finish(&mut self) -> io::Result<()> { + self.write_header()?; + self.inner.finish()?; + + while self.crc_bytes_written < 8 { + let (sum, amt) = (self.crc.sum() as u32, self.crc.amount()); + let buf = [ + (sum >> 0) as u8, + (sum >> 8) as u8, + (sum >> 16) as u8, + (sum >> 24) as u8, + (amt >> 0) as u8, + (amt >> 8) as u8, + (amt >> 16) as u8, + (amt >> 24) as u8, + ]; + let inner = self.inner.get_mut(); + let n = inner.write(&buf[self.crc_bytes_written..])?; + self.crc_bytes_written += n; + } + Ok(()) + } + + /// Finish encoding this stream, returning the underlying writer once the + /// encoding is done. + /// + /// Note that this function may not be suitable to call in a situation where + /// the underlying stream is an asynchronous I/O stream. To finish a stream + /// the `try_finish` (or `shutdown`) method should be used instead. To + /// re-acquire ownership of a stream it is safe to call this method after + /// `try_finish` or `shutdown` has returned `Ok`. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn finish(mut self) -> io::Result<W> { + self.try_finish()?; + Ok(self.inner.take_inner()) + } + + fn write_header(&mut self) -> io::Result<()> { + while self.header.len() > 0 { + let n = self.inner.get_mut().write(&self.header)?; + self.header.drain(..n); + } + Ok(()) + } +} + +impl<W: Write> Write for GzEncoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + assert_eq!(self.crc_bytes_written, 0); + self.write_header()?; + let n = self.inner.write(buf)?; + self.crc.update(&buf[..n]); + Ok(n) + } + + fn flush(&mut self) -> io::Result<()> { + assert_eq!(self.crc_bytes_written, 0); + self.write_header()?; + self.inner.flush() + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncWrite> AsyncWrite for GzEncoder<W> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.try_finish()?; + self.get_mut().shutdown() + } +} + +impl<R: Read + Write> Read for GzEncoder<R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.get_mut().read(buf) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + AsyncWrite> AsyncRead for GzEncoder<R> {} + +impl<W: Write> Drop for GzEncoder<W> { + fn drop(&mut self) { + if self.inner.is_present() { + let _ = self.try_finish(); + } + } +} + +/// A gzip streaming decoder +/// +/// This structure exposes a [`Write`] interface that will emit compressed data +/// to the underlying writer `W`. +/// +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// use flate2::Compression; +/// use flate2::write::{GzEncoder, GzDecoder}; +/// +/// # fn main() { +/// # let mut e = GzEncoder::new(Vec::new(), Compression::default()); +/// # e.write(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # assert_eq!("Hello World", decode_writer(bytes).unwrap()); +/// # } +/// // Uncompresses a gzip encoded vector of bytes and returns a string or error +/// // Here Vec<u8> implements Write +/// fn decode_writer(bytes: Vec<u8>) -> io::Result<String> { +/// let mut writer = Vec::new(); +/// let mut decoder = GzDecoder::new(writer); +/// decoder.write_all(&bytes[..])?; +/// writer = decoder.finish()?; +/// let return_string = String::from_utf8(writer).expect("String parsing error"); +/// Ok(return_string) +/// } +/// ``` +#[derive(Debug)] +pub struct GzDecoder<W: Write> { + inner: zio::Writer<CrcWriter<W>, Decompress>, + crc_bytes: Vec<u8>, + header: Option<GzHeader>, + header_buf: Vec<u8>, +} + +const CRC_BYTES_LEN: usize = 8; + +impl<W: Write> GzDecoder<W> { + /// Creates a new decoder which will write uncompressed data to the stream. + /// + /// When this encoder is dropped or unwrapped the final pieces of data will + /// be flushed. + pub fn new(w: W) -> GzDecoder<W> { + GzDecoder { + inner: zio::Writer::new(CrcWriter::new(w), Decompress::new(false)), + crc_bytes: Vec::with_capacity(CRC_BYTES_LEN), + header: None, + header_buf: Vec::new(), + } + } + + /// Returns the header associated with this stream. + pub fn header(&self) -> Option<&GzHeader> { + self.header.as_ref() + } + + /// Acquires a reference to the underlying writer. + pub fn get_ref(&self) -> &W { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying writer. + /// + /// Note that mutating the output/input state of the stream may corrupt this + /// object, so care must be taken when using this method. + pub fn get_mut(&mut self) -> &mut W { + self.inner.get_mut().get_mut() + } + + /// Attempt to finish this output stream, writing out final chunks of data. + /// + /// Note that this function can only be used once data has finished being + /// written to the output stream. After this function is called then further + /// calls to `write` may result in a panic. + /// + /// # Panics + /// + /// Attempts to write data to this stream may result in a panic after this + /// function is called. + /// + /// # Errors + /// + /// This function will perform I/O to finish the stream, returning any + /// errors which happen. + pub fn try_finish(&mut self) -> io::Result<()> { + self.finish_and_check_crc()?; + Ok(()) + } + + /// Consumes this decoder, flushing the output stream. + /// + /// This will flush the underlying data stream and then return the contained + /// writer if the flush succeeded. + /// + /// Note that this function may not be suitable to call in a situation where + /// the underlying stream is an asynchronous I/O stream. To finish a stream + /// the `try_finish` (or `shutdown`) method should be used instead. To + /// re-acquire ownership of a stream it is safe to call this method after + /// `try_finish` or `shutdown` has returned `Ok`. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn finish(mut self) -> io::Result<W> { + self.finish_and_check_crc()?; + Ok(self.inner.take_inner().into_inner()) + } + + fn finish_and_check_crc(&mut self) -> io::Result<()> { + self.inner.finish()?; + + if self.crc_bytes.len() != 8 { + return Err(corrupt()); + } + + let crc = ((self.crc_bytes[0] as u32) << 0) + | ((self.crc_bytes[1] as u32) << 8) + | ((self.crc_bytes[2] as u32) << 16) + | ((self.crc_bytes[3] as u32) << 24); + let amt = ((self.crc_bytes[4] as u32) << 0) + | ((self.crc_bytes[5] as u32) << 8) + | ((self.crc_bytes[6] as u32) << 16) + | ((self.crc_bytes[7] as u32) << 24); + if crc != self.inner.get_ref().crc().sum() as u32 { + return Err(corrupt()); + } + if amt != self.inner.get_ref().crc().amount() { + return Err(corrupt()); + } + Ok(()) + } +} + +struct Counter<T: Read> { + inner: T, + pos: usize, +} + +impl<T: Read> Read for Counter<T> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + let pos = self.inner.read(buf)?; + self.pos += pos; + Ok(pos) + } +} + +impl<W: Write> Write for GzDecoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + if self.header.is_none() { + // trying to avoid buffer usage + let (res, pos) = { + let mut counter = Counter { + inner: self.header_buf.chain(buf), + pos: 0, + }; + let res = read_gz_header(&mut counter); + (res, counter.pos) + }; + + match res { + Err(err) => { + if err.kind() == io::ErrorKind::UnexpectedEof { + // not enough data for header, save to the buffer + self.header_buf.extend(buf); + Ok(buf.len()) + } else { + Err(err) + } + } + Ok(header) => { + self.header = Some(header); + let pos = pos - self.header_buf.len(); + self.header_buf.truncate(0); + Ok(pos) + } + } + } else { + let (n, status) = self.inner.write_with_status(buf)?; + + if status == Status::StreamEnd { + if n < buf.len() && self.crc_bytes.len() < 8 { + let remaining = buf.len() - n; + let crc_bytes = cmp::min(remaining, CRC_BYTES_LEN - self.crc_bytes.len()); + self.crc_bytes.extend(&buf[n..n + crc_bytes]); + return Ok(n + crc_bytes); + } + } + Ok(n) + } + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncWrite> AsyncWrite for GzDecoder<W> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.try_finish()?; + self.inner.get_mut().get_mut().shutdown() + } +} + +impl<W: Read + Write> Read for GzDecoder<W> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.inner.get_mut().get_mut().read(buf) + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncRead + AsyncWrite> AsyncRead for GzDecoder<W> {} + +#[cfg(test)] +mod tests { + use super::*; + + const STR: &'static str = "Hello World Hello World Hello World Hello World Hello World \ + Hello World Hello World Hello World Hello World Hello World \ + Hello World Hello World Hello World Hello World Hello World \ + Hello World Hello World Hello World Hello World Hello World \ + Hello World Hello World Hello World Hello World Hello World"; + + #[test] + fn decode_writer_one_chunk() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write(STR.as_ref()).unwrap(); + let bytes = e.finish().unwrap(); + + let mut writer = Vec::new(); + let mut decoder = GzDecoder::new(writer); + let n = decoder.write(&bytes[..]).unwrap(); + decoder.write(&bytes[n..]).unwrap(); + decoder.try_finish().unwrap(); + writer = decoder.finish().unwrap(); + let return_string = String::from_utf8(writer).expect("String parsing error"); + assert_eq!(return_string, STR); + } + + #[test] + fn decode_writer_partial_header() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write(STR.as_ref()).unwrap(); + let bytes = e.finish().unwrap(); + + let mut writer = Vec::new(); + let mut decoder = GzDecoder::new(writer); + assert_eq!(decoder.write(&bytes[..5]).unwrap(), 5); + let n = decoder.write(&bytes[5..]).unwrap(); + if n < bytes.len() - 5 { + decoder.write(&bytes[n + 5..]).unwrap(); + } + writer = decoder.finish().unwrap(); + let return_string = String::from_utf8(writer).expect("String parsing error"); + assert_eq!(return_string, STR); + } + + #[test] + fn decode_writer_exact_header() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write(STR.as_ref()).unwrap(); + let bytes = e.finish().unwrap(); + + let mut writer = Vec::new(); + let mut decoder = GzDecoder::new(writer); + assert_eq!(decoder.write(&bytes[..10]).unwrap(), 10); + decoder.write(&bytes[10..]).unwrap(); + writer = decoder.finish().unwrap(); + let return_string = String::from_utf8(writer).expect("String parsing error"); + assert_eq!(return_string, STR); + } + + #[test] + fn decode_writer_partial_crc() { + let mut e = GzEncoder::new(Vec::new(), Compression::default()); + e.write(STR.as_ref()).unwrap(); + let bytes = e.finish().unwrap(); + + let mut writer = Vec::new(); + let mut decoder = GzDecoder::new(writer); + let l = bytes.len() - 5; + let n = decoder.write(&bytes[..l]).unwrap(); + decoder.write(&bytes[n..]).unwrap(); + writer = decoder.finish().unwrap(); + let return_string = String::from_utf8(writer).expect("String parsing error"); + assert_eq!(return_string, STR); + } +} diff --git a/third_party/rust/flate2/src/lib.rs b/third_party/rust/flate2/src/lib.rs new file mode 100644 index 0000000000..178803f4d5 --- /dev/null +++ b/third_party/rust/flate2/src/lib.rs @@ -0,0 +1,234 @@ +//! A DEFLATE-based stream compression/decompression library +//! +//! This library provides support for compression and decompression of +//! DEFLATE-based streams: +//! +//! * the DEFLATE format itself +//! * the zlib format +//! * gzip +//! +//! These three formats are all closely related and largely only differ in their +//! headers/footers. This crate has three types in each submodule for dealing +//! with these three formats. +//! +//! # Implementation +//! +//! In addition to supporting three formats, this crate supports three different +//! backends, controlled through this crate's features: +//! +//! * `default`, or `rust_backend` - this implementation uses the `miniz_oxide` +//! crate which is a port of `miniz.c` (below) to Rust. This feature does not +//! require a C compiler and only requires Rust code. +//! +//! * `miniz-sys` - when enabled this feature will enable this crate to instead +//! use `miniz.c`, distributed with `miniz-sys`, to implement +//! compression/decompression. +//! +//! * `zlib` - finally, this feature will enable linking against the `libz` +//! library, typically found on most Linux systems by default. If the library +//! isn't found to already be on the system it will be compiled from source +//! (this is a C library). +//! +//! There's various tradeoffs associated with each implementation, but in +//! general you probably won't have to tweak the defaults. The default choice is +//! selected to avoid the need for a C compiler at build time. The `miniz-sys` +//! feature is largely a historical artifact at this point and is unlikely to be +//! needed, and `zlib` is often useful if you're already using `zlib` for other +//! C dependencies. The compression ratios and performance of each of these +//! feature should be roughly comparable, but you'll likely want to run your own +//! tests if you're curious about the performance. +//! +//! # Organization +//! +//! This crate consists mainly of three modules, [`read`], [`write`], and +//! [`bufread`]. Each module contains a number of types used to encode and +//! decode various streams of data. +//! +//! All types in the [`write`] module work on instances of [`Write`][write], +//! whereas all types in the [`read`] module work on instances of +//! [`Read`][read] and [`bufread`] works with [`BufRead`][bufread]. If you +//! are decoding directly from a `&[u8]`, use the [`bufread`] types. +//! +//! ``` +//! use flate2::write::GzEncoder; +//! use flate2::Compression; +//! use std::io; +//! use std::io::prelude::*; +//! +//! # fn main() { let _ = run(); } +//! # fn run() -> io::Result<()> { +//! let mut encoder = GzEncoder::new(Vec::new(), Compression::default()); +//! encoder.write_all(b"Example")?; +//! # Ok(()) +//! # } +//! ``` +//! +//! +//! Other various types are provided at the top-level of the crate for +//! management and dealing with encoders/decoders. Also note that types which +//! operate over a specific trait often implement the mirroring trait as well. +//! For example a `flate2::read::DeflateDecoder<T>` *also* implements the +//! `Write` trait if `T: Write`. That is, the "dual trait" is forwarded directly +//! to the underlying object if available. +//! +//! [`read`]: read/index.html +//! [`bufread`]: bufread/index.html +//! [`write`]: write/index.html +//! [read]: https://doc.rust-lang.org/std/io/trait.Read.html +//! [write]: https://doc.rust-lang.org/std/io/trait.Write.html +//! [bufread]: https://doc.rust-lang.org/std/io/trait.BufRead.html +//! +//! # Async I/O +//! +//! This crate optionally can support async I/O streams with the [Tokio stack] via +//! the `tokio` feature of this crate: +//! +//! [Tokio stack]: https://tokio.rs/ +//! +//! ```toml +//! flate2 = { version = "0.2", features = ["tokio"] } +//! ``` +//! +//! All methods are internally capable of working with streams that may return +//! [`ErrorKind::WouldBlock`] when they're not ready to perform the particular +//! operation. +//! +//! [`ErrorKind::WouldBlock`]: https://doc.rust-lang.org/std/io/enum.ErrorKind.html +//! +//! Note that care needs to be taken when using these objects, however. The +//! Tokio runtime, in particular, requires that data is fully flushed before +//! dropping streams. For compatibility with blocking streams all streams are +//! flushed/written when they are dropped, and this is not always a suitable +//! time to perform I/O. If I/O streams are flushed before drop, however, then +//! these operations will be a noop. +#![doc(html_root_url = "https://docs.rs/flate2/0.2")] +#![deny(missing_docs)] +#![deny(missing_debug_implementations)] +#![allow(trivial_numeric_casts)] +#![cfg_attr(test, deny(warnings))] + +pub use crate::crc::{Crc, CrcReader, CrcWriter}; +pub use crate::gz::GzBuilder; +pub use crate::gz::GzHeader; +pub use crate::mem::{Compress, CompressError, Decompress, DecompressError, Status}; +pub use crate::mem::{FlushCompress, FlushDecompress}; + +mod bufreader; +mod crc; +mod deflate; +mod ffi; +mod gz; +mod mem; +mod zio; +mod zlib; + +/// Types which operate over [`Read`] streams, both encoders and decoders for +/// various formats. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +pub mod read { + pub use crate::deflate::read::DeflateDecoder; + pub use crate::deflate::read::DeflateEncoder; + pub use crate::gz::read::GzDecoder; + pub use crate::gz::read::GzEncoder; + pub use crate::gz::read::MultiGzDecoder; + pub use crate::zlib::read::ZlibDecoder; + pub use crate::zlib::read::ZlibEncoder; +} + +/// Types which operate over [`Write`] streams, both encoders and decoders for +/// various formats. +/// +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +pub mod write { + pub use crate::deflate::write::DeflateDecoder; + pub use crate::deflate::write::DeflateEncoder; + pub use crate::gz::write::GzDecoder; + pub use crate::gz::write::GzEncoder; + pub use crate::zlib::write::ZlibDecoder; + pub use crate::zlib::write::ZlibEncoder; +} + +/// Types which operate over [`BufRead`] streams, both encoders and decoders for +/// various formats. +/// +/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html +pub mod bufread { + pub use crate::deflate::bufread::DeflateDecoder; + pub use crate::deflate::bufread::DeflateEncoder; + pub use crate::gz::bufread::GzDecoder; + pub use crate::gz::bufread::GzEncoder; + pub use crate::gz::bufread::MultiGzDecoder; + pub use crate::zlib::bufread::ZlibDecoder; + pub use crate::zlib::bufread::ZlibEncoder; +} + +fn _assert_send_sync() { + fn _assert_send_sync<T: Send + Sync>() {} + + _assert_send_sync::<read::DeflateEncoder<&[u8]>>(); + _assert_send_sync::<read::DeflateDecoder<&[u8]>>(); + _assert_send_sync::<read::ZlibEncoder<&[u8]>>(); + _assert_send_sync::<read::ZlibDecoder<&[u8]>>(); + _assert_send_sync::<read::GzEncoder<&[u8]>>(); + _assert_send_sync::<read::GzDecoder<&[u8]>>(); + _assert_send_sync::<read::MultiGzDecoder<&[u8]>>(); + _assert_send_sync::<write::DeflateEncoder<Vec<u8>>>(); + _assert_send_sync::<write::DeflateDecoder<Vec<u8>>>(); + _assert_send_sync::<write::ZlibEncoder<Vec<u8>>>(); + _assert_send_sync::<write::ZlibDecoder<Vec<u8>>>(); + _assert_send_sync::<write::GzEncoder<Vec<u8>>>(); + _assert_send_sync::<write::GzDecoder<Vec<u8>>>(); +} + +/// When compressing data, the compression level can be specified by a value in +/// this enum. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct Compression(u32); + +impl Compression { + /// Creates a new description of the compression level with an explicitly + /// specified integer. + /// + /// The integer here is typically on a scale of 0-9 where 0 means "no + /// compression" and 9 means "take as long as you'd like". + pub const fn new(level: u32) -> Compression { + Compression(level) + } + + /// No compression is to be performed, this may actually inflate data + /// slightly when encoding. + pub const fn none() -> Compression { + Compression(0) + } + + /// Optimize for the best speed of encoding. + pub const fn fast() -> Compression { + Compression(1) + } + + /// Optimize for the size of data being encoded. + pub const fn best() -> Compression { + Compression(9) + } + + /// Returns an integer representing the compression level, typically on a + /// scale of 0-9 + pub fn level(&self) -> u32 { + self.0 + } +} + +impl Default for Compression { + fn default() -> Compression { + Compression(6) + } +} + +#[cfg(test)] +fn random_bytes() -> impl Iterator<Item = u8> { + use rand::Rng; + use std::iter; + + iter::repeat(()).map(|_| rand::thread_rng().gen()) +} diff --git a/third_party/rust/flate2/src/mem.rs b/third_party/rust/flate2/src/mem.rs new file mode 100644 index 0000000000..e293b4ba86 --- /dev/null +++ b/third_party/rust/flate2/src/mem.rs @@ -0,0 +1,648 @@ +use std::error::Error; +use std::fmt; +use std::io; +use std::slice; + +use crate::ffi::{self, Backend, Deflate, DeflateBackend, Inflate, InflateBackend}; +use crate::Compression; + +/// Raw in-memory compression stream for blocks of data. +/// +/// This type is the building block for the I/O streams in the rest of this +/// crate. It requires more management than the [`Read`]/[`Write`] API but is +/// maximally flexible in terms of accepting input from any source and being +/// able to produce output to any memory location. +/// +/// It is recommended to use the I/O stream adaptors over this type as they're +/// easier to use. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +#[derive(Debug)] +pub struct Compress { + inner: Deflate, +} + +/// Raw in-memory decompression stream for blocks of data. +/// +/// This type is the building block for the I/O streams in the rest of this +/// crate. It requires more management than the [`Read`]/[`Write`] API but is +/// maximally flexible in terms of accepting input from any source and being +/// able to produce output to any memory location. +/// +/// It is recommended to use the I/O stream adaptors over this type as they're +/// easier to use. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +#[derive(Debug)] +pub struct Decompress { + inner: Inflate, +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +/// Values which indicate the form of flushing to be used when compressing +/// in-memory data. +pub enum FlushCompress { + /// A typical parameter for passing to compression/decompression functions, + /// this indicates that the underlying stream to decide how much data to + /// accumulate before producing output in order to maximize compression. + None = ffi::MZ_NO_FLUSH as isize, + + /// All pending output is flushed to the output buffer and the output is + /// aligned on a byte boundary so that the decompressor can get all input + /// data available so far. + /// + /// Flushing may degrade compression for some compression algorithms and so + /// it should only be used when necessary. This will complete the current + /// deflate block and follow it with an empty stored block. + Sync = ffi::MZ_SYNC_FLUSH as isize, + + /// All pending output is flushed to the output buffer, but the output is + /// not aligned to a byte boundary. + /// + /// All of the input data so far will be available to the decompressor (as + /// with `Flush::Sync`. This completes the current deflate block and follows + /// it with an empty fixed codes block that is 10 bites long, and it assures + /// that enough bytes are output in order for the decompessor to finish the + /// block before the empty fixed code block. + Partial = ffi::MZ_PARTIAL_FLUSH as isize, + + /// All output is flushed as with `Flush::Sync` and the compression state is + /// reset so decompression can restart from this point if previous + /// compressed data has been damaged or if random access is desired. + /// + /// Using this option too often can seriously degrade compression. + Full = ffi::MZ_FULL_FLUSH as isize, + + /// Pending input is processed and pending output is flushed. + /// + /// The return value may indicate that the stream is not yet done and more + /// data has yet to be processed. + Finish = ffi::MZ_FINISH as isize, + + #[doc(hidden)] + _Nonexhaustive, +} + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +/// Values which indicate the form of flushing to be used when +/// decompressing in-memory data. +pub enum FlushDecompress { + /// A typical parameter for passing to compression/decompression functions, + /// this indicates that the underlying stream to decide how much data to + /// accumulate before producing output in order to maximize compression. + None = ffi::MZ_NO_FLUSH as isize, + + /// All pending output is flushed to the output buffer and the output is + /// aligned on a byte boundary so that the decompressor can get all input + /// data available so far. + /// + /// Flushing may degrade compression for some compression algorithms and so + /// it should only be used when necessary. This will complete the current + /// deflate block and follow it with an empty stored block. + Sync = ffi::MZ_SYNC_FLUSH as isize, + + /// Pending input is processed and pending output is flushed. + /// + /// The return value may indicate that the stream is not yet done and more + /// data has yet to be processed. + Finish = ffi::MZ_FINISH as isize, + + #[doc(hidden)] + _Nonexhaustive, +} + +/// The inner state for an error when decompressing +#[derive(Debug, Default)] +pub(crate) struct DecompressErrorInner { + pub(crate) needs_dictionary: Option<u32>, +} + +/// Error returned when a decompression object finds that the input stream of +/// bytes was not a valid input stream of bytes. +#[derive(Debug)] +pub struct DecompressError(pub(crate) DecompressErrorInner); + +impl DecompressError { + /// Indicates whether decompression failed due to requiring a dictionary. + /// + /// The resulting integer is the Adler-32 checksum of the dictionary + /// required. + pub fn needs_dictionary(&self) -> Option<u32> { + self.0.needs_dictionary + } +} + +#[inline] +pub(crate) fn decompress_failed() -> Result<Status, DecompressError> { + Err(DecompressError(Default::default())) +} + +#[inline] +pub(crate) fn decompress_need_dict(adler: u32) -> Result<Status, DecompressError> { + Err(DecompressError(DecompressErrorInner { + needs_dictionary: Some(adler), + })) +} + +/// Error returned when a compression object is used incorrectly or otherwise +/// generates an error. +#[derive(Debug)] +pub struct CompressError(pub(crate) ()); + +/// Possible status results of compressing some data or successfully +/// decompressing a block of data. +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub enum Status { + /// Indicates success. + /// + /// Means that more input may be needed but isn't available + /// and/or there's more output to be written but the output buffer is full. + Ok, + + /// Indicates that forward progress is not possible due to input or output + /// buffers being empty. + /// + /// For compression it means the input buffer needs some more data or the + /// output buffer needs to be freed up before trying again. + /// + /// For decompression this means that more input is needed to continue or + /// the output buffer isn't large enough to contain the result. The function + /// can be called again after fixing both. + BufError, + + /// Indicates that all input has been consumed and all output bytes have + /// been written. Decompression/compression should not be called again. + /// + /// For decompression with zlib streams the adler-32 of the decompressed + /// data has also been verified. + StreamEnd, +} + +impl Compress { + /// Creates a new object ready for compressing data that it's given. + /// + /// The `level` argument here indicates what level of compression is going + /// to be performed, and the `zlib_header` argument indicates whether the + /// output data should have a zlib header or not. + pub fn new(level: Compression, zlib_header: bool) -> Compress { + Compress { + inner: Deflate::make(level, zlib_header, ffi::MZ_DEFAULT_WINDOW_BITS as u8), + } + } + + /// Creates a new object ready for compressing data that it's given. + /// + /// The `level` argument here indicates what level of compression is going + /// to be performed, and the `zlib_header` argument indicates whether the + /// output data should have a zlib header or not. The `window_bits` parameter + /// indicates the base-2 logarithm of the sliding window size and must be + /// between 9 and 15. + /// + /// # Panics + /// + /// If `window_bits` does not fall into the range 9 ..= 15, + /// `new_with_window_bits` will panic. + /// + /// # Note + /// + /// This constructor is only available when the `zlib` feature is used. + /// Other backends currently do not support custom window bits. + #[cfg(feature = "any_zlib")] + pub fn new_with_window_bits( + level: Compression, + zlib_header: bool, + window_bits: u8, + ) -> Compress { + Compress { + inner: Deflate::make(level, zlib_header, window_bits), + } + } + + /// Returns the total number of input bytes which have been processed by + /// this compression object. + pub fn total_in(&self) -> u64 { + self.inner.total_in() + } + + /// Returns the total number of output bytes which have been produced by + /// this compression object. + pub fn total_out(&self) -> u64 { + self.inner.total_out() + } + + /// Specifies the compression dictionary to use. + /// + /// Returns the Adler-32 checksum of the dictionary. + #[cfg(feature = "any_zlib")] + pub fn set_dictionary(&mut self, dictionary: &[u8]) -> Result<u32, CompressError> { + let stream = &mut *self.inner.inner.stream_wrapper; + let rc = unsafe { + assert!(dictionary.len() < ffi::uInt::max_value() as usize); + ffi::deflateSetDictionary(stream, dictionary.as_ptr(), dictionary.len() as ffi::uInt) + }; + + match rc { + ffi::MZ_STREAM_ERROR => Err(CompressError(())), + ffi::MZ_OK => Ok(stream.adler as u32), + c => panic!("unknown return code: {}", c), + } + } + + /// Quickly resets this compressor without having to reallocate anything. + /// + /// This is equivalent to dropping this object and then creating a new one. + pub fn reset(&mut self) { + self.inner.reset(); + } + + /// Dynamically updates the compression level. + /// + /// This can be used to switch between compression levels for different + /// kinds of data, or it can be used in conjunction with a call to reset + /// to reuse the compressor. + /// + /// This may return an error if there wasn't enough output space to complete + /// the compression of the available input data before changing the + /// compression level. Flushing the stream before calling this method + /// ensures that the function will succeed on the first call. + #[cfg(feature = "any_zlib")] + pub fn set_level(&mut self, level: Compression) -> Result<(), CompressError> { + use libc::c_int; + let stream = &mut *self.inner.inner.stream_wrapper; + + let rc = unsafe { ffi::deflateParams(stream, level.0 as c_int, ffi::MZ_DEFAULT_STRATEGY) }; + + match rc { + ffi::MZ_OK => Ok(()), + ffi::MZ_BUF_ERROR => Err(CompressError(())), + c => panic!("unknown return code: {}", c), + } + } + + /// Compresses the input data into the output, consuming only as much + /// input as needed and writing as much output as possible. + /// + /// The flush option can be any of the available `FlushCompress` parameters. + /// + /// To learn how much data was consumed or how much output was produced, use + /// the `total_in` and `total_out` functions before/after this is called. + pub fn compress( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushCompress, + ) -> Result<Status, CompressError> { + self.inner.compress(input, output, flush) + } + + /// Compresses the input data into the extra space of the output, consuming + /// only as much input as needed and writing as much output as possible. + /// + /// This function has the same semantics as `compress`, except that the + /// length of `vec` is managed by this function. This will not reallocate + /// the vector provided or attempt to grow it, so space for the output must + /// be reserved in the output vector by the caller before calling this + /// function. + pub fn compress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: FlushCompress, + ) -> Result<Status, CompressError> { + let cap = output.capacity(); + let len = output.len(); + + unsafe { + let before = self.total_out(); + let ret = { + let ptr = output.as_mut_ptr().offset(len as isize); + let out = slice::from_raw_parts_mut(ptr, cap - len); + self.compress(input, out, flush) + }; + output.set_len((self.total_out() - before) as usize + len); + return ret; + } + } +} + +impl Decompress { + /// Creates a new object ready for decompressing data that it's given. + /// + /// The `zlib_header` argument indicates whether the input data is expected + /// to have a zlib header or not. + pub fn new(zlib_header: bool) -> Decompress { + Decompress { + inner: Inflate::make(zlib_header, ffi::MZ_DEFAULT_WINDOW_BITS as u8), + } + } + + /// Creates a new object ready for decompressing data that it's given. + /// + /// The `zlib_header` argument indicates whether the input data is expected + /// to have a zlib header or not. The `window_bits` parameter indicates the + /// base-2 logarithm of the sliding window size and must be between 9 and 15. + /// + /// # Panics + /// + /// If `window_bits` does not fall into the range 9 ..= 15, + /// `new_with_window_bits` will panic. + /// + /// # Note + /// + /// This constructor is only available when the `zlib` feature is used. + /// Other backends currently do not support custom window bits. + #[cfg(feature = "any_zlib")] + pub fn new_with_window_bits(zlib_header: bool, window_bits: u8) -> Decompress { + Decompress { + inner: Inflate::make(zlib_header, window_bits), + } + } + + /// Returns the total number of input bytes which have been processed by + /// this decompression object. + pub fn total_in(&self) -> u64 { + self.inner.total_in() + } + + /// Returns the total number of output bytes which have been produced by + /// this decompression object. + pub fn total_out(&self) -> u64 { + self.inner.total_out() + } + + /// Decompresses the input data into the output, consuming only as much + /// input as needed and writing as much output as possible. + /// + /// The flush option can be any of the available `FlushDecompress` parameters. + /// + /// If the first call passes `FlushDecompress::Finish` it is assumed that + /// the input and output buffers are both sized large enough to decompress + /// the entire stream in a single call. + /// + /// A flush value of `FlushDecompress::Finish` indicates that there are no + /// more source bytes available beside what's already in the input buffer, + /// and the output buffer is large enough to hold the rest of the + /// decompressed data. + /// + /// To learn how much data was consumed or how much output was produced, use + /// the `total_in` and `total_out` functions before/after this is called. + /// + /// # Errors + /// + /// If the input data to this instance of `Decompress` is not a valid + /// zlib/deflate stream then this function may return an instance of + /// `DecompressError` to indicate that the stream of input bytes is corrupted. + pub fn decompress( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushDecompress, + ) -> Result<Status, DecompressError> { + self.inner.decompress(input, output, flush) + } + + /// Decompresses the input data into the extra space in the output vector + /// specified by `output`. + /// + /// This function has the same semantics as `decompress`, except that the + /// length of `vec` is managed by this function. This will not reallocate + /// the vector provided or attempt to grow it, so space for the output must + /// be reserved in the output vector by the caller before calling this + /// function. + /// + /// # Errors + /// + /// If the input data to this instance of `Decompress` is not a valid + /// zlib/deflate stream then this function may return an instance of + /// `DecompressError` to indicate that the stream of input bytes is corrupted. + pub fn decompress_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: FlushDecompress, + ) -> Result<Status, DecompressError> { + let cap = output.capacity(); + let len = output.len(); + + unsafe { + let before = self.total_out(); + let ret = { + let ptr = output.as_mut_ptr().offset(len as isize); + let out = slice::from_raw_parts_mut(ptr, cap - len); + self.decompress(input, out, flush) + }; + output.set_len((self.total_out() - before) as usize + len); + return ret; + } + } + + /// Specifies the decompression dictionary to use. + #[cfg(feature = "any_zlib")] + pub fn set_dictionary(&mut self, dictionary: &[u8]) -> Result<u32, DecompressError> { + let stream = &mut *self.inner.inner.stream_wrapper; + let rc = unsafe { + assert!(dictionary.len() < ffi::uInt::max_value() as usize); + ffi::inflateSetDictionary(stream, dictionary.as_ptr(), dictionary.len() as ffi::uInt) + }; + + match rc { + ffi::MZ_STREAM_ERROR => Err(DecompressError(Default::default())), + ffi::MZ_DATA_ERROR => Err(DecompressError(DecompressErrorInner { + needs_dictionary: Some(stream.adler as u32), + })), + ffi::MZ_OK => Ok(stream.adler as u32), + c => panic!("unknown return code: {}", c), + } + } + + /// Performs the equivalent of replacing this decompression state with a + /// freshly allocated copy. + /// + /// This function may not allocate memory, though, and attempts to reuse any + /// previously existing resources. + /// + /// The argument provided here indicates whether the reset state will + /// attempt to decode a zlib header first or not. + pub fn reset(&mut self, zlib_header: bool) { + self.inner.reset(zlib_header); + } +} + +impl Error for DecompressError {} + +impl From<DecompressError> for io::Error { + fn from(data: DecompressError) -> io::Error { + io::Error::new(io::ErrorKind::Other, data) + } +} + +impl fmt::Display for DecompressError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "deflate decompression error") + } +} + +impl Error for CompressError {} + +impl From<CompressError> for io::Error { + fn from(data: CompressError) -> io::Error { + io::Error::new(io::ErrorKind::Other, data) + } +} + +impl fmt::Display for CompressError { + fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { + write!(f, "deflate decompression error") + } +} + +#[cfg(test)] +mod tests { + use std::io::Write; + + use crate::write; + use crate::{Compression, Decompress, FlushDecompress}; + + #[cfg(feature = "any_zlib")] + use crate::{Compress, FlushCompress}; + + #[test] + fn issue51() { + let data = vec![ + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0xb3, 0xc9, 0x28, 0xc9, + 0xcd, 0xb1, 0xe3, 0xe5, 0xb2, 0xc9, 0x48, 0x4d, 0x4c, 0xb1, 0xb3, 0x29, 0xc9, 0x2c, + 0xc9, 0x49, 0xb5, 0x33, 0x31, 0x30, 0x51, 0xf0, 0xcb, 0x2f, 0x51, 0x70, 0xcb, 0x2f, + 0xcd, 0x4b, 0xb1, 0xd1, 0x87, 0x08, 0xda, 0xe8, 0x83, 0x95, 0x00, 0x95, 0x26, 0xe5, + 0xa7, 0x54, 0x2a, 0x24, 0xa5, 0x27, 0xe7, 0xe7, 0xe4, 0x17, 0xd9, 0x2a, 0x95, 0x67, + 0x64, 0x96, 0xa4, 0x2a, 0x81, 0x8c, 0x48, 0x4e, 0xcd, 0x2b, 0x49, 0x2d, 0xb2, 0xb3, + 0xc9, 0x30, 0x44, 0x37, 0x01, 0x28, 0x62, 0xa3, 0x0f, 0x95, 0x06, 0xd9, 0x05, 0x54, + 0x04, 0xe5, 0xe5, 0xa5, 0x67, 0xe6, 0x55, 0xe8, 0x1b, 0xea, 0x99, 0xe9, 0x19, 0x21, + 0xab, 0xd0, 0x07, 0xd9, 0x01, 0x32, 0x53, 0x1f, 0xea, 0x3e, 0x00, 0x94, 0x85, 0xeb, + 0xe4, 0xa8, 0x00, 0x00, 0x00, + ]; + + let mut decoded = Vec::with_capacity(data.len() * 2); + + let mut d = Decompress::new(false); + // decompressed whole deflate stream + assert!(d + .decompress_vec(&data[10..], &mut decoded, FlushDecompress::Finish) + .is_ok()); + + // decompress data that has nothing to do with the deflate stream (this + // used to panic) + drop(d.decompress_vec(&[0], &mut decoded, FlushDecompress::None)); + } + + #[test] + fn reset() { + let string = "hello world".as_bytes(); + let mut zlib = Vec::new(); + let mut deflate = Vec::new(); + + let comp = Compression::default(); + write::ZlibEncoder::new(&mut zlib, comp) + .write_all(string) + .unwrap(); + write::DeflateEncoder::new(&mut deflate, comp) + .write_all(string) + .unwrap(); + + let mut dst = [0; 1024]; + let mut decoder = Decompress::new(true); + decoder + .decompress(&zlib, &mut dst, FlushDecompress::Finish) + .unwrap(); + assert_eq!(decoder.total_out(), string.len() as u64); + assert!(dst.starts_with(string)); + + decoder.reset(false); + decoder + .decompress(&deflate, &mut dst, FlushDecompress::Finish) + .unwrap(); + assert_eq!(decoder.total_out(), string.len() as u64); + assert!(dst.starts_with(string)); + } + + #[cfg(feature = "any_zlib")] + #[test] + fn set_dictionary_with_zlib_header() { + let string = "hello, hello!".as_bytes(); + let dictionary = "hello".as_bytes(); + + let mut encoded = Vec::with_capacity(1024); + + let mut encoder = Compress::new(Compression::default(), true); + + let dictionary_adler = encoder.set_dictionary(&dictionary).unwrap(); + + encoder + .compress_vec(string, &mut encoded, FlushCompress::Finish) + .unwrap(); + + assert_eq!(encoder.total_in(), string.len() as u64); + assert_eq!(encoder.total_out(), encoded.len() as u64); + + let mut decoder = Decompress::new(true); + let mut decoded = [0; 1024]; + let decompress_error = decoder + .decompress(&encoded, &mut decoded, FlushDecompress::Finish) + .expect_err("decompression should fail due to requiring a dictionary"); + + let required_adler = decompress_error.needs_dictionary() + .expect("the first call to decompress should indicate a dictionary is required along with the required Adler-32 checksum"); + + assert_eq!(required_adler, dictionary_adler, + "the Adler-32 checksum should match the value when the dictionary was set on the compressor"); + + let actual_adler = decoder.set_dictionary(&dictionary).unwrap(); + + assert_eq!(required_adler, actual_adler); + + // Decompress the rest of the input to the remainder of the output buffer + let total_in = decoder.total_in(); + let total_out = decoder.total_out(); + + let decompress_result = decoder.decompress( + &encoded[total_in as usize..], + &mut decoded[total_out as usize..], + FlushDecompress::Finish, + ); + assert!(decompress_result.is_ok()); + + assert_eq!(&decoded[..decoder.total_out() as usize], string); + } + + #[cfg(feature = "any_zlib")] + #[test] + fn set_dictionary_raw() { + let string = "hello, hello!".as_bytes(); + let dictionary = "hello".as_bytes(); + + let mut encoded = Vec::with_capacity(1024); + + let mut encoder = Compress::new(Compression::default(), false); + + encoder.set_dictionary(&dictionary).unwrap(); + + encoder + .compress_vec(string, &mut encoded, FlushCompress::Finish) + .unwrap(); + + assert_eq!(encoder.total_in(), string.len() as u64); + assert_eq!(encoder.total_out(), encoded.len() as u64); + + let mut decoder = Decompress::new(false); + + decoder.set_dictionary(&dictionary).unwrap(); + + let mut decoded = [0; 1024]; + let decompress_result = decoder.decompress(&encoded, &mut decoded, FlushDecompress::Finish); + + assert!(decompress_result.is_ok()); + + assert_eq!(&decoded[..decoder.total_out() as usize], string); + } +} diff --git a/third_party/rust/flate2/src/zio.rs b/third_party/rust/flate2/src/zio.rs new file mode 100644 index 0000000000..50281885bd --- /dev/null +++ b/third_party/rust/flate2/src/zio.rs @@ -0,0 +1,290 @@ +use std::io; +use std::io::prelude::*; +use std::mem; + +use crate::{Compress, Decompress, DecompressError, FlushCompress, FlushDecompress, Status}; + +#[derive(Debug)] +pub struct Writer<W: Write, D: Ops> { + obj: Option<W>, + pub data: D, + buf: Vec<u8>, +} + +pub trait Ops { + type Flush: Flush; + fn total_in(&self) -> u64; + fn total_out(&self) -> u64; + fn run( + &mut self, + input: &[u8], + output: &mut [u8], + flush: Self::Flush, + ) -> Result<Status, DecompressError>; + fn run_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: Self::Flush, + ) -> Result<Status, DecompressError>; +} + +impl Ops for Compress { + type Flush = FlushCompress; + fn total_in(&self) -> u64 { + self.total_in() + } + fn total_out(&self) -> u64 { + self.total_out() + } + fn run( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushCompress, + ) -> Result<Status, DecompressError> { + Ok(self.compress(input, output, flush).unwrap()) + } + fn run_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: FlushCompress, + ) -> Result<Status, DecompressError> { + Ok(self.compress_vec(input, output, flush).unwrap()) + } +} + +impl Ops for Decompress { + type Flush = FlushDecompress; + fn total_in(&self) -> u64 { + self.total_in() + } + fn total_out(&self) -> u64 { + self.total_out() + } + fn run( + &mut self, + input: &[u8], + output: &mut [u8], + flush: FlushDecompress, + ) -> Result<Status, DecompressError> { + self.decompress(input, output, flush) + } + fn run_vec( + &mut self, + input: &[u8], + output: &mut Vec<u8>, + flush: FlushDecompress, + ) -> Result<Status, DecompressError> { + self.decompress_vec(input, output, flush) + } +} + +pub trait Flush { + fn none() -> Self; + fn sync() -> Self; + fn finish() -> Self; +} + +impl Flush for FlushCompress { + fn none() -> Self { + FlushCompress::None + } + + fn sync() -> Self { + FlushCompress::Sync + } + + fn finish() -> Self { + FlushCompress::Finish + } +} + +impl Flush for FlushDecompress { + fn none() -> Self { + FlushDecompress::None + } + + fn sync() -> Self { + FlushDecompress::Sync + } + + fn finish() -> Self { + FlushDecompress::Finish + } +} + +pub fn read<R, D>(obj: &mut R, data: &mut D, dst: &mut [u8]) -> io::Result<usize> +where + R: BufRead, + D: Ops, +{ + loop { + let (read, consumed, ret, eof); + { + let input = obj.fill_buf()?; + eof = input.is_empty(); + let before_out = data.total_out(); + let before_in = data.total_in(); + let flush = if eof { + D::Flush::finish() + } else { + D::Flush::none() + }; + ret = data.run(input, dst, flush); + read = (data.total_out() - before_out) as usize; + consumed = (data.total_in() - before_in) as usize; + } + obj.consume(consumed); + + match ret { + // If we haven't ready any data and we haven't hit EOF yet, + // then we need to keep asking for more data because if we + // return that 0 bytes of data have been read then it will + // be interpreted as EOF. + Ok(Status::Ok) | Ok(Status::BufError) if read == 0 && !eof && dst.len() > 0 => continue, + Ok(Status::Ok) | Ok(Status::BufError) | Ok(Status::StreamEnd) => return Ok(read), + + Err(..) => { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "corrupt deflate stream", + )) + } + } + } +} + +impl<W: Write, D: Ops> Writer<W, D> { + pub fn new(w: W, d: D) -> Writer<W, D> { + Writer { + obj: Some(w), + data: d, + buf: Vec::with_capacity(32 * 1024), + } + } + + pub fn finish(&mut self) -> io::Result<()> { + loop { + self.dump()?; + + let before = self.data.total_out(); + self.data.run_vec(&[], &mut self.buf, D::Flush::finish())?; + if before == self.data.total_out() { + return Ok(()); + } + } + } + + pub fn replace(&mut self, w: W) -> W { + self.buf.truncate(0); + mem::replace(self.get_mut(), w) + } + + pub fn get_ref(&self) -> &W { + self.obj.as_ref().unwrap() + } + + pub fn get_mut(&mut self) -> &mut W { + self.obj.as_mut().unwrap() + } + + // Note that this should only be called if the outer object is just about + // to be consumed! + // + // (e.g. an implementation of `into_inner`) + pub fn take_inner(&mut self) -> W { + self.obj.take().unwrap() + } + + pub fn is_present(&self) -> bool { + self.obj.is_some() + } + + // Returns total written bytes and status of underlying codec + pub(crate) fn write_with_status(&mut self, buf: &[u8]) -> io::Result<(usize, Status)> { + // miniz isn't guaranteed to actually write any of the buffer provided, + // it may be in a flushing mode where it's just giving us data before + // we're actually giving it any data. We don't want to spuriously return + // `Ok(0)` when possible as it will cause calls to write_all() to fail. + // As a result we execute this in a loop to ensure that we try our + // darndest to write the data. + loop { + self.dump()?; + + let before_in = self.data.total_in(); + let ret = self.data.run_vec(buf, &mut self.buf, D::Flush::none()); + let written = (self.data.total_in() - before_in) as usize; + + let is_stream_end = match ret { + Ok(Status::StreamEnd) => true, + _ => false, + }; + + if buf.len() > 0 && written == 0 && ret.is_ok() && !is_stream_end { + continue; + } + return match ret { + Ok(st) => match st { + Status::Ok | Status::BufError | Status::StreamEnd => Ok((written, st)), + }, + Err(..) => Err(io::Error::new( + io::ErrorKind::InvalidInput, + "corrupt deflate stream", + )), + }; + } + } + + fn dump(&mut self) -> io::Result<()> { + // TODO: should manage this buffer not with `drain` but probably more of + // a deque-like strategy. + while self.buf.len() > 0 { + let n = self.obj.as_mut().unwrap().write(&self.buf)?; + if n == 0 { + return Err(io::ErrorKind::WriteZero.into()); + } + self.buf.drain(..n); + } + Ok(()) + } +} + +impl<W: Write, D: Ops> Write for Writer<W, D> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.write_with_status(buf).map(|res| res.0) + } + + fn flush(&mut self) -> io::Result<()> { + self.data + .run_vec(&[], &mut self.buf, D::Flush::sync()) + .unwrap(); + + // Unfortunately miniz doesn't actually tell us when we're done with + // pulling out all the data from the internal stream. To remedy this we + // have to continually ask the stream for more memory until it doesn't + // give us a chunk of memory the same size as our own internal buffer, + // at which point we assume it's reached the end. + loop { + self.dump()?; + let before = self.data.total_out(); + self.data + .run_vec(&[], &mut self.buf, D::Flush::none()) + .unwrap(); + if before == self.data.total_out() { + break; + } + } + + self.obj.as_mut().unwrap().flush() + } +} + +impl<W: Write, D: Ops> Drop for Writer<W, D> { + fn drop(&mut self) { + if self.obj.is_some() { + let _ = self.finish(); + } + } +} diff --git a/third_party/rust/flate2/src/zlib/bufread.rs b/third_party/rust/flate2/src/zlib/bufread.rs new file mode 100644 index 0000000000..182f0647dc --- /dev/null +++ b/third_party/rust/flate2/src/zlib/bufread.rs @@ -0,0 +1,258 @@ +use std::io; +use std::io::prelude::*; +use std::mem; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use crate::zio; +use crate::{Compress, Decompress}; + +/// A ZLIB encoder, or compressor. +/// +/// This structure consumes a [`BufRead`] interface, reading uncompressed data +/// from the underlying reader, and emitting compressed data. +/// +/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use flate2::Compression; +/// use flate2::bufread::ZlibEncoder; +/// use std::fs::File; +/// use std::io::BufReader; +/// +/// // Use a buffered file to compress contents into a Vec<u8> +/// +/// # fn open_hello_world() -> std::io::Result<Vec<u8>> { +/// let f = File::open("examples/hello_world.txt")?; +/// let b = BufReader::new(f); +/// let mut z = ZlibEncoder::new(b, Compression::fast()); +/// let mut buffer = Vec::new(); +/// z.read_to_end(&mut buffer)?; +/// # Ok(buffer) +/// # } +/// ``` +#[derive(Debug)] +pub struct ZlibEncoder<R> { + obj: R, + data: Compress, +} + +impl<R: BufRead> ZlibEncoder<R> { + /// Creates a new encoder which will read uncompressed data from the given + /// stream and emit the compressed stream. + pub fn new(r: R, level: crate::Compression) -> ZlibEncoder<R> { + ZlibEncoder { + obj: r, + data: Compress::new(level, true), + } + } +} + +pub fn reset_encoder_data<R>(zlib: &mut ZlibEncoder<R>) { + zlib.data.reset() +} + +impl<R> ZlibEncoder<R> { + /// Resets the state of this encoder entirely, swapping out the input + /// stream for another. + /// + /// This function will reset the internal state of this encoder and replace + /// the input stream with the one provided, returning the previous input + /// stream. Future data read from this encoder will be the compressed + /// version of `r`'s data. + pub fn reset(&mut self, r: R) -> R { + reset_encoder_data(self); + mem::replace(&mut self.obj, r) + } + + /// Acquires a reference to the underlying reader + pub fn get_ref(&self) -> &R { + &self.obj + } + + /// Acquires a mutable reference to the underlying stream + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + &mut self.obj + } + + /// Consumes this encoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.obj + } + + /// Returns the number of bytes that have been read into this compressor. + /// + /// Note that not all bytes read from the underlying object may be accounted + /// for, there may still be some active buffering. + pub fn total_in(&self) -> u64 { + self.data.total_in() + } + + /// Returns the number of bytes that the compressor has produced. + /// + /// Note that not all bytes may have been read yet, some may still be + /// buffered. + pub fn total_out(&self) -> u64 { + self.data.total_out() + } +} + +impl<R: BufRead> Read for ZlibEncoder<R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + zio::read(&mut self.obj, &mut self.data, buf) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + BufRead> AsyncRead for ZlibEncoder<R> {} + +impl<R: BufRead + Write> Write for ZlibEncoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + BufRead> AsyncWrite for ZlibEncoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} + +/// A ZLIB decoder, or decompressor. +/// +/// This structure consumes a [`BufRead`] interface, reading compressed data +/// from the underlying reader, and emitting uncompressed data. +/// +/// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::ZlibEncoder; +/// use flate2::bufread::ZlibDecoder; +/// +/// # fn main() { +/// # let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_bufreader(bytes).unwrap()); +/// # } +/// # +/// // Uncompresses a Zlib Encoded vector of bytes and returns a string or error +/// // Here &[u8] implements BufRead +/// +/// fn decode_bufreader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut z = ZlibDecoder::new(&bytes[..]); +/// let mut s = String::new(); +/// z.read_to_string(&mut s)?; +/// Ok(s) +/// } +/// ``` +#[derive(Debug)] +pub struct ZlibDecoder<R> { + obj: R, + data: Decompress, +} + +impl<R: BufRead> ZlibDecoder<R> { + /// Creates a new decoder which will decompress data read from the given + /// stream. + pub fn new(r: R) -> ZlibDecoder<R> { + ZlibDecoder { + obj: r, + data: Decompress::new(true), + } + } +} + +pub fn reset_decoder_data<R>(zlib: &mut ZlibDecoder<R>) { + zlib.data = Decompress::new(true); +} + +impl<R> ZlibDecoder<R> { + /// Resets the state of this decoder entirely, swapping out the input + /// stream for another. + /// + /// This will reset the internal state of this decoder and replace the + /// input stream with the one provided, returning the previous input + /// stream. Future data read from this decoder will be the decompressed + /// version of `r`'s data. + pub fn reset(&mut self, r: R) -> R { + reset_decoder_data(self); + mem::replace(&mut self.obj, r) + } + + /// Acquires a reference to the underlying stream + pub fn get_ref(&self) -> &R { + &self.obj + } + + /// Acquires a mutable reference to the underlying stream + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + &mut self.obj + } + + /// Consumes this decoder, returning the underlying reader. + pub fn into_inner(self) -> R { + self.obj + } + + /// Returns the number of bytes that the decompressor has consumed. + /// + /// Note that this will likely be smaller than what the decompressor + /// actually read from the underlying stream due to buffering. + pub fn total_in(&self) -> u64 { + self.data.total_in() + } + + /// Returns the number of bytes that the decompressor has produced. + pub fn total_out(&self) -> u64 { + self.data.total_out() + } +} + +impl<R: BufRead> Read for ZlibDecoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + zio::read(&mut self.obj, &mut self.data, into) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + BufRead> AsyncRead for ZlibDecoder<R> {} + +impl<R: BufRead + Write> Write for ZlibDecoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + BufRead> AsyncWrite for ZlibDecoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} diff --git a/third_party/rust/flate2/src/zlib/mod.rs b/third_party/rust/flate2/src/zlib/mod.rs new file mode 100644 index 0000000000..1813a4e2ac --- /dev/null +++ b/third_party/rust/flate2/src/zlib/mod.rs @@ -0,0 +1,159 @@ +pub mod bufread; +pub mod read; +pub mod write; + +#[cfg(test)] +mod tests { + use std::io; + use std::io::prelude::*; + + use rand::{thread_rng, Rng}; + + use crate::zlib::{read, write}; + use crate::Compression; + + #[test] + fn roundtrip() { + let mut real = Vec::new(); + let mut w = write::ZlibEncoder::new(Vec::new(), Compression::default()); + let v = crate::random_bytes().take(1024).collect::<Vec<_>>(); + for _ in 0..200 { + let to_write = &v[..thread_rng().gen_range(0, v.len())]; + real.extend(to_write.iter().map(|x| *x)); + w.write_all(to_write).unwrap(); + } + let result = w.finish().unwrap(); + let mut r = read::ZlibDecoder::new(&result[..]); + let mut ret = Vec::new(); + r.read_to_end(&mut ret).unwrap(); + assert!(ret == real); + } + + #[test] + fn drop_writes() { + let mut data = Vec::new(); + write::ZlibEncoder::new(&mut data, Compression::default()) + .write_all(b"foo") + .unwrap(); + let mut r = read::ZlibDecoder::new(&data[..]); + let mut ret = Vec::new(); + r.read_to_end(&mut ret).unwrap(); + assert!(ret == b"foo"); + } + + #[test] + fn total_in() { + let mut real = Vec::new(); + let mut w = write::ZlibEncoder::new(Vec::new(), Compression::default()); + let v = crate::random_bytes().take(1024).collect::<Vec<_>>(); + for _ in 0..200 { + let to_write = &v[..thread_rng().gen_range(0, v.len())]; + real.extend(to_write.iter().map(|x| *x)); + w.write_all(to_write).unwrap(); + } + let mut result = w.finish().unwrap(); + + let result_len = result.len(); + + for _ in 0..200 { + result.extend(v.iter().map(|x| *x)); + } + + let mut r = read::ZlibDecoder::new(&result[..]); + let mut ret = Vec::new(); + r.read_to_end(&mut ret).unwrap(); + assert!(ret == real); + assert_eq!(r.total_in(), result_len as u64); + } + + #[test] + fn roundtrip2() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let mut r = read::ZlibDecoder::new(read::ZlibEncoder::new(&v[..], Compression::default())); + let mut ret = Vec::new(); + r.read_to_end(&mut ret).unwrap(); + assert_eq!(ret, v); + } + + #[test] + fn roundtrip3() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let mut w = + write::ZlibEncoder::new(write::ZlibDecoder::new(Vec::new()), Compression::default()); + w.write_all(&v).unwrap(); + let w = w.finish().unwrap().finish().unwrap(); + assert!(w == v); + } + + #[test] + fn reset_decoder() { + let v = crate::random_bytes().take(1024 * 1024).collect::<Vec<_>>(); + let mut w = write::ZlibEncoder::new(Vec::new(), Compression::default()); + w.write_all(&v).unwrap(); + let data = w.finish().unwrap(); + + { + let (mut a, mut b, mut c) = (Vec::new(), Vec::new(), Vec::new()); + let mut r = read::ZlibDecoder::new(&data[..]); + r.read_to_end(&mut a).unwrap(); + r.reset(&data); + r.read_to_end(&mut b).unwrap(); + + let mut r = read::ZlibDecoder::new(&data[..]); + r.read_to_end(&mut c).unwrap(); + assert!(a == b && b == c && c == v); + } + + { + let mut w = write::ZlibDecoder::new(Vec::new()); + w.write_all(&data).unwrap(); + let a = w.reset(Vec::new()).unwrap(); + w.write_all(&data).unwrap(); + let b = w.finish().unwrap(); + + let mut w = write::ZlibDecoder::new(Vec::new()); + w.write_all(&data).unwrap(); + let c = w.finish().unwrap(); + assert!(a == b && b == c && c == v); + } + } + + #[test] + fn bad_input() { + // regress tests: previously caused a panic on drop + let mut out: Vec<u8> = Vec::new(); + let data: Vec<u8> = (0..255).cycle().take(1024).collect(); + let mut w = write::ZlibDecoder::new(&mut out); + match w.write_all(&data[..]) { + Ok(_) => panic!("Expected an error to be returned!"), + Err(e) => assert_eq!(e.kind(), io::ErrorKind::InvalidInput), + } + } + + #[test] + fn qc_reader() { + ::quickcheck::quickcheck(test as fn(_) -> _); + + fn test(v: Vec<u8>) -> bool { + let mut r = + read::ZlibDecoder::new(read::ZlibEncoder::new(&v[..], Compression::default())); + let mut v2 = Vec::new(); + r.read_to_end(&mut v2).unwrap(); + v == v2 + } + } + + #[test] + fn qc_writer() { + ::quickcheck::quickcheck(test as fn(_) -> _); + + fn test(v: Vec<u8>) -> bool { + let mut w = write::ZlibEncoder::new( + write::ZlibDecoder::new(Vec::new()), + Compression::default(), + ); + w.write_all(&v).unwrap(); + v == w.finish().unwrap().finish().unwrap() + } + } +} diff --git a/third_party/rust/flate2/src/zlib/read.rs b/third_party/rust/flate2/src/zlib/read.rs new file mode 100644 index 0000000000..43f350246c --- /dev/null +++ b/third_party/rust/flate2/src/zlib/read.rs @@ -0,0 +1,265 @@ +use std::io; +use std::io::prelude::*; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use super::bufread; +use crate::bufreader::BufReader; + +/// A ZLIB encoder, or compressor. +/// +/// This structure implements a [`Read`] interface and will read uncompressed +/// data from an underlying stream and emit a stream of compressed data. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use flate2::Compression; +/// use flate2::read::ZlibEncoder; +/// use std::fs::File; +/// +/// // Open example file and compress the contents using Read interface +/// +/// # fn open_hello_world() -> std::io::Result<Vec<u8>> { +/// let f = File::open("examples/hello_world.txt")?; +/// let mut z = ZlibEncoder::new(f, Compression::fast()); +/// let mut buffer = [0;50]; +/// let byte_count = z.read(&mut buffer)?; +/// # Ok(buffer[0..byte_count].to_vec()) +/// # } +/// ``` +#[derive(Debug)] +pub struct ZlibEncoder<R> { + inner: bufread::ZlibEncoder<BufReader<R>>, +} + +impl<R: Read> ZlibEncoder<R> { + /// Creates a new encoder which will read uncompressed data from the given + /// stream and emit the compressed stream. + pub fn new(r: R, level: crate::Compression) -> ZlibEncoder<R> { + ZlibEncoder { + inner: bufread::ZlibEncoder::new(BufReader::new(r), level), + } + } +} + +impl<R> ZlibEncoder<R> { + /// Resets the state of this encoder entirely, swapping out the input + /// stream for another. + /// + /// This function will reset the internal state of this encoder and replace + /// the input stream with the one provided, returning the previous input + /// stream. Future data read from this encoder will be the compressed + /// version of `r`'s data. + /// + /// Note that there may be currently buffered data when this function is + /// called, and in that case the buffered data is discarded. + pub fn reset(&mut self, r: R) -> R { + super::bufread::reset_encoder_data(&mut self.inner); + self.inner.get_mut().reset(r) + } + + /// Acquires a reference to the underlying stream + pub fn get_ref(&self) -> &R { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying stream + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.inner.get_mut().get_mut() + } + + /// Consumes this encoder, returning the underlying reader. + /// + /// Note that there may be buffered bytes which are not re-acquired as part + /// of this transition. It's recommended to only call this function after + /// EOF has been reached. + pub fn into_inner(self) -> R { + self.inner.into_inner().into_inner() + } + + /// Returns the number of bytes that have been read into this compressor. + /// + /// Note that not all bytes read from the underlying object may be accounted + /// for, there may still be some active buffering. + pub fn total_in(&self) -> u64 { + self.inner.total_in() + } + + /// Returns the number of bytes that the compressor has produced. + /// + /// Note that not all bytes may have been read yet, some may still be + /// buffered. + pub fn total_out(&self) -> u64 { + self.inner.total_out() + } +} + +impl<R: Read> Read for ZlibEncoder<R> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.inner.read(buf) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead> AsyncRead for ZlibEncoder<R> {} + +impl<W: Read + Write> Write for ZlibEncoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead + AsyncWrite> AsyncWrite for ZlibEncoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} + +/// A ZLIB decoder, or decompressor. +/// +/// This structure implements a [`Read`] interface and takes a stream of +/// compressed data as input, providing the decompressed data when read from. +/// +/// [`Read`]: https://doc.rust-lang.org/std/io/trait.Read.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::ZlibEncoder; +/// use flate2::read::ZlibDecoder; +/// +/// # fn main() { +/// # let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_reader(bytes).unwrap()); +/// # } +/// # +/// // Uncompresses a Zlib Encoded vector of bytes and returns a string or error +/// // Here &[u8] implements Read +/// +/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut z = ZlibDecoder::new(&bytes[..]); +/// let mut s = String::new(); +/// z.read_to_string(&mut s)?; +/// Ok(s) +/// } +/// ``` +#[derive(Debug)] +pub struct ZlibDecoder<R> { + inner: bufread::ZlibDecoder<BufReader<R>>, +} + +impl<R: Read> ZlibDecoder<R> { + /// Creates a new decoder which will decompress data read from the given + /// stream. + pub fn new(r: R) -> ZlibDecoder<R> { + ZlibDecoder::new_with_buf(r, vec![0; 32 * 1024]) + } + + /// Same as `new`, but the intermediate buffer for data is specified. + /// + /// Note that the specified buffer will only be used up to its current + /// length. The buffer's capacity will also not grow over time. + pub fn new_with_buf(r: R, buf: Vec<u8>) -> ZlibDecoder<R> { + ZlibDecoder { + inner: bufread::ZlibDecoder::new(BufReader::with_buf(buf, r)), + } + } +} + +impl<R> ZlibDecoder<R> { + /// Resets the state of this decoder entirely, swapping out the input + /// stream for another. + /// + /// This will reset the internal state of this decoder and replace the + /// input stream with the one provided, returning the previous input + /// stream. Future data read from this decoder will be the decompressed + /// version of `r`'s data. + /// + /// Note that there may be currently buffered data when this function is + /// called, and in that case the buffered data is discarded. + pub fn reset(&mut self, r: R) -> R { + super::bufread::reset_decoder_data(&mut self.inner); + self.inner.get_mut().reset(r) + } + + /// Acquires a reference to the underlying stream + pub fn get_ref(&self) -> &R { + self.inner.get_ref().get_ref() + } + + /// Acquires a mutable reference to the underlying stream + /// + /// Note that mutation of the stream may result in surprising results if + /// this encoder is continued to be used. + pub fn get_mut(&mut self) -> &mut R { + self.inner.get_mut().get_mut() + } + + /// Consumes this decoder, returning the underlying reader. + /// + /// Note that there may be buffered bytes which are not re-acquired as part + /// of this transition. It's recommended to only call this function after + /// EOF has been reached. + pub fn into_inner(self) -> R { + self.inner.into_inner().into_inner() + } + + /// Returns the number of bytes that the decompressor has consumed. + /// + /// Note that this will likely be smaller than what the decompressor + /// actually read from the underlying stream due to buffering. + pub fn total_in(&self) -> u64 { + self.inner.total_in() + } + + /// Returns the number of bytes that the decompressor has produced. + pub fn total_out(&self) -> u64 { + self.inner.total_out() + } +} + +impl<R: Read> Read for ZlibDecoder<R> { + fn read(&mut self, into: &mut [u8]) -> io::Result<usize> { + self.inner.read(into) + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncRead> AsyncRead for ZlibDecoder<R> {} + +impl<R: Read + Write> Write for ZlibDecoder<R> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.get_mut().write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.get_mut().flush() + } +} + +#[cfg(feature = "tokio")] +impl<R: AsyncWrite + AsyncRead> AsyncWrite for ZlibDecoder<R> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.get_mut().shutdown() + } +} diff --git a/third_party/rust/flate2/src/zlib/write.rs b/third_party/rust/flate2/src/zlib/write.rs new file mode 100644 index 0000000000..7135c343c9 --- /dev/null +++ b/third_party/rust/flate2/src/zlib/write.rs @@ -0,0 +1,348 @@ +use std::io; +use std::io::prelude::*; + +#[cfg(feature = "tokio")] +use futures::Poll; +#[cfg(feature = "tokio")] +use tokio_io::{AsyncRead, AsyncWrite}; + +use crate::zio; +use crate::{Compress, Decompress}; + +/// A ZLIB encoder, or compressor. +/// +/// This structure implements a [`Write`] interface and takes a stream of +/// uncompressed data, writing the compressed data to the wrapped writer. +/// +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use flate2::Compression; +/// use flate2::write::ZlibEncoder; +/// +/// // Vec<u8> implements Write, assigning the compressed bytes of sample string +/// +/// # fn zlib_encoding() -> std::io::Result<()> { +/// let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); +/// e.write_all(b"Hello World")?; +/// let compressed = e.finish()?; +/// # Ok(()) +/// # } +/// ``` +#[derive(Debug)] +pub struct ZlibEncoder<W: Write> { + inner: zio::Writer<W, Compress>, +} + +impl<W: Write> ZlibEncoder<W> { + /// Creates a new encoder which will write compressed data to the stream + /// given at the given compression level. + /// + /// When this encoder is dropped or unwrapped the final pieces of data will + /// be flushed. + pub fn new(w: W, level: crate::Compression) -> ZlibEncoder<W> { + ZlibEncoder { + inner: zio::Writer::new(w, Compress::new(level, true)), + } + } + + /// Acquires a reference to the underlying writer. + pub fn get_ref(&self) -> &W { + self.inner.get_ref() + } + + /// Acquires a mutable reference to the underlying writer. + /// + /// Note that mutating the output/input state of the stream may corrupt this + /// object, so care must be taken when using this method. + pub fn get_mut(&mut self) -> &mut W { + self.inner.get_mut() + } + + /// Resets the state of this encoder entirely, swapping out the output + /// stream for another. + /// + /// This function will finish encoding the current stream into the current + /// output stream before swapping out the two output streams. + /// + /// After the current stream has been finished, this will reset the internal + /// state of this encoder and replace the output stream with the one + /// provided, returning the previous output stream. Future data written to + /// this encoder will be the compressed into the stream `w` provided. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn reset(&mut self, w: W) -> io::Result<W> { + self.inner.finish()?; + self.inner.data.reset(); + Ok(self.inner.replace(w)) + } + + /// Attempt to finish this output stream, writing out final chunks of data. + /// + /// Note that this function can only be used once data has finished being + /// written to the output stream. After this function is called then further + /// calls to `write` may result in a panic. + /// + /// # Panics + /// + /// Attempts to write data to this stream may result in a panic after this + /// function is called. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn try_finish(&mut self) -> io::Result<()> { + self.inner.finish() + } + + /// Consumes this encoder, flushing the output stream. + /// + /// This will flush the underlying data stream, close off the compressed + /// stream and, if successful, return the contained writer. + /// + /// Note that this function may not be suitable to call in a situation where + /// the underlying stream is an asynchronous I/O stream. To finish a stream + /// the `try_finish` (or `shutdown`) method should be used instead. To + /// re-acquire ownership of a stream it is safe to call this method after + /// `try_finish` or `shutdown` has returned `Ok`. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn finish(mut self) -> io::Result<W> { + self.inner.finish()?; + Ok(self.inner.take_inner()) + } + + /// Consumes this encoder, flushing the output stream. + /// + /// This will flush the underlying data stream and then return the contained + /// writer if the flush succeeded. + /// The compressed stream will not closed but only flushed. This + /// means that obtained byte array can by extended by another deflated + /// stream. To close the stream add the two bytes 0x3 and 0x0. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn flush_finish(mut self) -> io::Result<W> { + self.inner.flush()?; + Ok(self.inner.take_inner()) + } + + /// Returns the number of bytes that have been written to this compresor. + /// + /// Note that not all bytes written to this object may be accounted for, + /// there may still be some active buffering. + pub fn total_in(&self) -> u64 { + self.inner.data.total_in() + } + + /// Returns the number of bytes that the compressor has produced. + /// + /// Note that not all bytes may have been written yet, some may still be + /// buffered. + pub fn total_out(&self) -> u64 { + self.inner.data.total_out() + } +} + +impl<W: Write> Write for ZlibEncoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.inner.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncWrite> AsyncWrite for ZlibEncoder<W> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.try_finish()?; + self.get_mut().shutdown() + } +} + +impl<W: Read + Write> Read for ZlibEncoder<W> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.get_mut().read(buf) + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncRead + AsyncWrite> AsyncRead for ZlibEncoder<W> {} + +/// A ZLIB decoder, or decompressor. +/// +/// This structure implements a [`Write`] and will emit a stream of decompressed +/// data when fed a stream of compressed data. +/// +/// [`Write`]: https://doc.rust-lang.org/std/io/trait.Write.html +/// +/// # Examples +/// +/// ``` +/// use std::io::prelude::*; +/// use std::io; +/// # use flate2::Compression; +/// # use flate2::write::ZlibEncoder; +/// use flate2::write::ZlibDecoder; +/// +/// # fn main() { +/// # let mut e = ZlibEncoder::new(Vec::new(), Compression::default()); +/// # e.write_all(b"Hello World").unwrap(); +/// # let bytes = e.finish().unwrap(); +/// # println!("{}", decode_reader(bytes).unwrap()); +/// # } +/// # +/// // Uncompresses a Zlib Encoded vector of bytes and returns a string or error +/// // Here Vec<u8> implements Write +/// +/// fn decode_reader(bytes: Vec<u8>) -> io::Result<String> { +/// let mut writer = Vec::new(); +/// let mut z = ZlibDecoder::new(writer); +/// z.write_all(&bytes[..])?; +/// writer = z.finish()?; +/// let return_string = String::from_utf8(writer).expect("String parsing error"); +/// Ok(return_string) +/// } +/// ``` +#[derive(Debug)] +pub struct ZlibDecoder<W: Write> { + inner: zio::Writer<W, Decompress>, +} + +impl<W: Write> ZlibDecoder<W> { + /// Creates a new decoder which will write uncompressed data to the stream. + /// + /// When this decoder is dropped or unwrapped the final pieces of data will + /// be flushed. + pub fn new(w: W) -> ZlibDecoder<W> { + ZlibDecoder { + inner: zio::Writer::new(w, Decompress::new(true)), + } + } + + /// Acquires a reference to the underlying writer. + pub fn get_ref(&self) -> &W { + self.inner.get_ref() + } + + /// Acquires a mutable reference to the underlying writer. + /// + /// Note that mutating the output/input state of the stream may corrupt this + /// object, so care must be taken when using this method. + pub fn get_mut(&mut self) -> &mut W { + self.inner.get_mut() + } + + /// Resets the state of this decoder entirely, swapping out the output + /// stream for another. + /// + /// This will reset the internal state of this decoder and replace the + /// output stream with the one provided, returning the previous output + /// stream. Future data written to this decoder will be decompressed into + /// the output stream `w`. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn reset(&mut self, w: W) -> io::Result<W> { + self.inner.finish()?; + self.inner.data = Decompress::new(true); + Ok(self.inner.replace(w)) + } + + /// Attempt to finish this output stream, writing out final chunks of data. + /// + /// Note that this function can only be used once data has finished being + /// written to the output stream. After this function is called then further + /// calls to `write` may result in a panic. + /// + /// # Panics + /// + /// Attempts to write data to this stream may result in a panic after this + /// function is called. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn try_finish(&mut self) -> io::Result<()> { + self.inner.finish() + } + + /// Consumes this encoder, flushing the output stream. + /// + /// This will flush the underlying data stream and then return the contained + /// writer if the flush succeeded. + /// + /// Note that this function may not be suitable to call in a situation where + /// the underlying stream is an asynchronous I/O stream. To finish a stream + /// the `try_finish` (or `shutdown`) method should be used instead. To + /// re-acquire ownership of a stream it is safe to call this method after + /// `try_finish` or `shutdown` has returned `Ok`. + /// + /// # Errors + /// + /// This function will perform I/O to complete this stream, and any I/O + /// errors which occur will be returned from this function. + pub fn finish(mut self) -> io::Result<W> { + self.inner.finish()?; + Ok(self.inner.take_inner()) + } + + /// Returns the number of bytes that the decompressor has consumed for + /// decompression. + /// + /// Note that this will likely be smaller than the number of bytes + /// successfully written to this stream due to internal buffering. + pub fn total_in(&self) -> u64 { + self.inner.data.total_in() + } + + /// Returns the number of bytes that the decompressor has written to its + /// output stream. + pub fn total_out(&self) -> u64 { + self.inner.data.total_out() + } +} + +impl<W: Write> Write for ZlibDecoder<W> { + fn write(&mut self, buf: &[u8]) -> io::Result<usize> { + self.inner.write(buf) + } + + fn flush(&mut self) -> io::Result<()> { + self.inner.flush() + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncWrite> AsyncWrite for ZlibDecoder<W> { + fn shutdown(&mut self) -> Poll<(), io::Error> { + self.inner.finish()?; + self.inner.get_mut().shutdown() + } +} + +impl<W: Read + Write> Read for ZlibDecoder<W> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.inner.get_mut().read(buf) + } +} + +#[cfg(feature = "tokio")] +impl<W: AsyncRead + AsyncWrite> AsyncRead for ZlibDecoder<W> {} diff --git a/third_party/rust/flate2/tests/async-reader.rs b/third_party/rust/flate2/tests/async-reader.rs new file mode 100644 index 0000000000..16dae65f91 --- /dev/null +++ b/third_party/rust/flate2/tests/async-reader.rs @@ -0,0 +1,96 @@ +extern crate flate2; +extern crate futures; +extern crate tokio_io; + +use flate2::read::{GzDecoder, MultiGzDecoder}; +use futures::prelude::*; +use futures::task; +use std::cmp; +use std::fs::File; +use std::io::{self, Read}; +use tokio_io::io::read_to_end; +use tokio_io::AsyncRead; + +struct BadReader<T> { + reader: T, + x: bool, +} + +impl<T> BadReader<T> { + fn new(reader: T) -> BadReader<T> { + BadReader { reader, x: true } + } +} + +impl<T: Read> Read for BadReader<T> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + if self.x { + self.x = false; + let len = cmp::min(buf.len(), 1); + self.reader.read(&mut buf[..len]) + } else { + self.x = true; + Err(io::ErrorKind::WouldBlock.into()) + } + } +} + +struct AssertAsync<T>(T); + +impl<T: Read> Read for AssertAsync<T> { + fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> { + self.0.read(buf) + } +} + +impl<T: Read> AsyncRead for AssertAsync<T> {} + +struct AlwaysNotify<T>(T); + +impl<T: Future> Future for AlwaysNotify<T> { + type Item = T::Item; + type Error = T::Error; + + fn poll(&mut self) -> Poll<Self::Item, Self::Error> { + let ret = self.0.poll(); + if let Ok(Async::NotReady) = &ret { + task::current().notify(); + } + ret + } +} + +#[test] +fn test_gz_asyncread() { + let f = File::open("tests/good-file.gz").unwrap(); + + let fut = read_to_end(AssertAsync(GzDecoder::new(BadReader::new(f))), Vec::new()); + let (_, content) = AlwaysNotify(fut).wait().unwrap(); + + let mut expected = Vec::new(); + File::open("tests/good-file.txt") + .unwrap() + .read_to_end(&mut expected) + .unwrap(); + + assert_eq!(content, expected); +} + +#[test] +fn test_multi_gz_asyncread() { + let f = File::open("tests/multi.gz").unwrap(); + + let fut = read_to_end( + AssertAsync(MultiGzDecoder::new(BadReader::new(f))), + Vec::new(), + ); + let (_, content) = AlwaysNotify(fut).wait().unwrap(); + + let mut expected = Vec::new(); + File::open("tests/multi.txt") + .unwrap() + .read_to_end(&mut expected) + .unwrap(); + + assert_eq!(content, expected); +} diff --git a/third_party/rust/flate2/tests/corrupt-file.gz b/third_party/rust/flate2/tests/corrupt-file.gz Binary files differnew file mode 100644 index 0000000000..159333b032 --- /dev/null +++ b/third_party/rust/flate2/tests/corrupt-file.gz diff --git a/third_party/rust/flate2/tests/early-flush.rs b/third_party/rust/flate2/tests/early-flush.rs new file mode 100644 index 0000000000..e717adaa52 --- /dev/null +++ b/third_party/rust/flate2/tests/early-flush.rs @@ -0,0 +1,20 @@ +extern crate flate2; + +use std::io::{Read, Write}; + +use flate2::read::GzDecoder; +use flate2::write::GzEncoder; + +#[test] +fn smoke() { + let mut w = GzEncoder::new(Vec::new(), flate2::Compression::default()); + w.flush().unwrap(); + w.write_all(b"hello").unwrap(); + + let bytes = w.finish().unwrap(); + + let mut r = GzDecoder::new(&bytes[..]); + let mut s = String::new(); + r.read_to_string(&mut s).unwrap(); + assert_eq!(s, "hello"); +} diff --git a/third_party/rust/flate2/tests/empty-read.rs b/third_party/rust/flate2/tests/empty-read.rs new file mode 100644 index 0000000000..7551238336 --- /dev/null +++ b/third_party/rust/flate2/tests/empty-read.rs @@ -0,0 +1,82 @@ +extern crate flate2; + +use std::io::{Read, Write}; + +#[test] +fn deflate_decoder_empty_read() { + let original: &[u8] = b"Lorem ipsum dolor sit amet."; + let mut encoder = + flate2::write::DeflateEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(original).unwrap(); + let encoded: Vec<u8> = encoder.finish().unwrap(); + let mut decoder = flate2::read::DeflateDecoder::new(encoded.as_slice()); + assert_eq!(decoder.read(&mut []).unwrap(), 0); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), original); +} + +#[test] +fn deflate_encoder_empty_read() { + let original: &[u8] = b"Lorem ipsum dolor sit amet."; + let mut encoder = flate2::read::DeflateEncoder::new(original, flate2::Compression::default()); + assert_eq!(encoder.read(&mut []).unwrap(), 0); + let mut encoded = Vec::new(); + encoder.read_to_end(&mut encoded).unwrap(); + let mut decoder = flate2::read::DeflateDecoder::new(encoded.as_slice()); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), original); +} + +#[test] +fn gzip_decoder_empty_read() { + let original: &[u8] = b"Lorem ipsum dolor sit amet."; + let mut encoder = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(original).unwrap(); + let encoded: Vec<u8> = encoder.finish().unwrap(); + let mut decoder = flate2::read::GzDecoder::new(encoded.as_slice()); + assert_eq!(decoder.read(&mut []).unwrap(), 0); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), original); +} + +#[test] +fn gzip_encoder_empty_read() { + let original: &[u8] = b"Lorem ipsum dolor sit amet."; + let mut encoder = flate2::read::GzEncoder::new(original, flate2::Compression::default()); + assert_eq!(encoder.read(&mut []).unwrap(), 0); + let mut encoded = Vec::new(); + encoder.read_to_end(&mut encoded).unwrap(); + let mut decoder = flate2::read::GzDecoder::new(encoded.as_slice()); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), original); +} + +#[test] +fn zlib_decoder_empty_read() { + let original: &[u8] = b"Lorem ipsum dolor sit amet."; + let mut encoder = flate2::write::ZlibEncoder::new(Vec::new(), flate2::Compression::default()); + encoder.write_all(original).unwrap(); + let encoded: Vec<u8> = encoder.finish().unwrap(); + let mut decoder = flate2::read::ZlibDecoder::new(encoded.as_slice()); + assert_eq!(decoder.read(&mut []).unwrap(), 0); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), original); +} + +#[test] +fn zlib_encoder_empty_read() { + let original: &[u8] = b"Lorem ipsum dolor sit amet."; + let mut encoder = flate2::read::ZlibEncoder::new(original, flate2::Compression::default()); + assert_eq!(encoder.read(&mut []).unwrap(), 0); + let mut encoded = Vec::new(); + encoder.read_to_end(&mut encoded).unwrap(); + let mut decoder = flate2::read::ZlibDecoder::new(encoded.as_slice()); + let mut decoded = Vec::new(); + decoder.read_to_end(&mut decoded).unwrap(); + assert_eq!(decoded.as_slice(), original); +} diff --git a/third_party/rust/flate2/tests/good-file.gz b/third_party/rust/flate2/tests/good-file.gz Binary files differnew file mode 100644 index 0000000000..f968689cce --- /dev/null +++ b/third_party/rust/flate2/tests/good-file.gz diff --git a/third_party/rust/flate2/tests/good-file.txt b/third_party/rust/flate2/tests/good-file.txt new file mode 100644 index 0000000000..ee39ac53db --- /dev/null +++ b/third_party/rust/flate2/tests/good-file.txt @@ -0,0 +1,733 @@ +## ## +timestep simulated EIR patent hosts +0 0.136402 16855 +1 0.146872 18564 +2 0.150157 20334 +3 0.146358 22159 +4 0.136315 23655 +5 0.122354 24848 +6 0.104753 25887 +7 0.084439 26770 +8 0.06417 27238 +9 0.0450397 27349 +10 0.0295473 27274 +11 0.0184662 26909 +12 0.0110032 26324 +13 0.00634348 25513 +14 0.0036144 24469 +15 0.00208133 23383 +16 0.00122468 22345 +17 0.000752514 21342 +18 0.000545333 20416 +19 0.000546139 19657 +20 0.00054572 18806 +21 0.000545757 18015 +22 0.000545898 17349 +23 0.000546719 16594 +24 0.000547353 15955 +25 0.000547944 15374 +26 0.000547606 14765 +27 0.000594773 14212 +28 0.000969163 13677 +29 0.00168295 13180 +30 0.003059 12760 +31 0.00571599 12313 +32 0.0107918 11896 +33 0.0201943 11512 +34 0.0368013 11340 +35 0.0640629 11323 +36 0.104447 11769 +37 0.157207 12728 +38 0.216682 14261 +39 0.271159 16491 +40 0.303552 19274 +41 0.303678 22157 +42 0.271945 24875 +43 0.215445 27027 +44 0.154503 28690 +45 0.100717 30046 +46 0.0600343 30602 +47 0.0328576 30709 +48 0.016964 30315 +49 0.00841526 29310 +50 0.0040958 28058 +51 0.0019953 26662 +52 0.000986531 25259 +53 0.000545786 24049 +54 0.000546405 22966 +55 0.000546036 21933 +56 0.00054427 20953 +57 0.000542769 20057 +58 0.000541566 19304 +59 0.000541822 18477 +60 0.000541643 17695 +61 0.000541989 17002 +62 0.000769298 16391 +63 0.00150811 15805 +64 0.00295097 15172 +65 0.00566197 14690 +66 0.0105243 14206 +67 0.0186965 13791 +68 0.0313363 13470 +69 0.0490605 13377 +70 0.0711679 13631 +71 0.0953625 14209 +72 0.118026 15277 +73 0.134612 16760 +74 0.144311 18339 +75 0.146328 20124 +76 0.142936 21803 +77 0.134029 23435 +78 0.120562 24854 +79 0.103157 25880 +80 0.0834054 26597 +81 0.0632474 27226 +82 0.0447785 27294 +83 0.0295654 27169 +84 0.0184081 26803 +85 0.0109489 26265 +86 0.00631234 25375 +87 0.00359978 24306 +88 0.00206967 23260 +89 0.00122197 22225 +90 0.000751031 21277 +91 0.000544507 20295 +92 0.000543897 19417 +93 0.000543483 18623 +94 0.000542926 17837 +95 0.000542685 17070 +96 0.000542387 16424 +97 0.000541194 15838 +98 0.000540427 15177 +99 0.000540774 14608 +100 0.000588312 14066 +101 0.000959183 13499 +102 0.00166774 12979 +103 0.00303278 12545 +104 0.00567457 12067 +105 0.0107272 11712 +106 0.0200606 11368 +107 0.0364637 11207 +108 0.063339 11238 +109 0.103717 11660 +110 0.156884 12621 +111 0.217072 14151 +112 0.272311 16358 +113 0.305046 19005 +114 0.304927 21926 +115 0.272427 24662 +116 0.216478 27080 +117 0.155168 29064 +118 0.10079 30370 +119 0.0599659 30992 +120 0.0331287 30975 +121 0.017235 30317 +122 0.00860221 29455 +123 0.00419286 28172 +124 0.00203361 26809 +125 0.000998847 25476 +126 0.000551418 24230 +127 0.000551119 23106 +128 0.000552786 22147 +129 0.000553814 21183 +130 0.000553743 20280 +131 0.000554428 19423 +132 0.000555022 18598 +133 0.000555921 17864 +134 0.000556687 17187 +135 0.000789996 16527 +136 0.00154597 15870 +137 0.00302776 15226 +138 0.00581484 14685 +139 0.010812 14234 +140 0.0191832 13818 +141 0.0321572 13571 +142 0.050328 13538 +143 0.072817 13812 +144 0.0974321 14368 +145 0.120225 15436 +146 0.137418 16988 +147 0.147086 18775 +148 0.149165 20563 +149 0.144943 22223 +150 0.136631 23741 +151 0.123355 24920 +152 0.105401 25779 +153 0.0851918 26781 +154 0.0641702 27265 +155 0.0450746 27505 +156 0.0294136 27416 +157 0.0183811 27028 +158 0.0109285 26260 +159 0.00634296 25451 +160 0.00364513 24472 +161 0.0021051 23427 +162 0.00123693 22403 +163 0.000759531 21393 +164 0.000551727 20485 +165 0.000552256 19660 +166 0.000552303 18862 +167 0.000550927 18094 +168 0.000551098 17378 +169 0.000551093 16691 +170 0.000551885 16050 +171 0.000552282 15420 +172 0.000552591 14878 +173 0.00060109 14357 +174 0.000980446 13768 +175 0.00170301 13241 +176 0.003096 12745 +177 0.00579971 12294 +178 0.010976 11879 +179 0.0205422 11636 +180 0.0374515 11431 +181 0.0649916 11517 +182 0.106008 11966 +183 0.159983 12918 +184 0.221127 14484 +185 0.276503 16696 +186 0.310316 19518 +187 0.311205 22301 +188 0.276769 25047 +189 0.220506 27360 +190 0.159123 29133 +191 0.103761 30440 +192 0.0613797 31087 +193 0.033583 31037 +194 0.0173275 30555 +195 0.00861968 29617 +196 0.00419503 28292 +197 0.00203304 26944 +198 0.00100126 25569 +199 0.000553511 24349 +200 0.000554687 23257 +201 0.00055586 22204 +202 0.000555419 21176 +203 0.000556032 20316 +204 0.000555974 19509 +205 0.000556859 18746 +206 0.000556996 17978 +207 0.000557102 17288 +208 0.000790187 16672 +209 0.00154711 16057 +210 0.00303521 15449 +211 0.00584201 14915 +212 0.0108854 14397 +213 0.0193386 14010 +214 0.0324346 13730 +215 0.0507192 13674 +216 0.0736661 13874 +217 0.0987887 14515 +218 0.122411 15693 +219 0.139964 17265 +220 0.149125 18894 +221 0.151434 20662 +222 0.148067 22442 +223 0.138894 24116 +224 0.125436 25367 +225 0.107664 26360 +226 0.0865709 27044 +227 0.0655588 27428 +228 0.0459664 27714 +229 0.0301384 27687 +230 0.0186481 27262 +231 0.01103 26677 +232 0.00636957 25722 +233 0.00366188 24662 +234 0.00212213 23575 +235 0.00125358 22520 +236 0.000768665 21480 +237 0.000556393 20563 +238 0.000555892 19706 +239 0.00055534 18914 +240 0.000555027 18165 +241 0.000555062 17432 +242 0.000553766 16733 +243 0.000552984 16070 +244 0.000553634 15396 +245 0.000554286 14867 +246 0.000603759 14362 +247 0.000982974 13867 +248 0.00170532 13379 +249 0.00310471 12907 +250 0.00582577 12446 +251 0.0110122 12018 +252 0.0206284 11730 +253 0.0375835 11546 +254 0.0652192 11605 +255 0.10646 11981 +256 0.160858 12949 +257 0.223122 14478 +258 0.279678 16810 +259 0.312171 19452 +260 0.311778 22391 +261 0.276966 25204 +262 0.22251 27379 +263 0.159246 29248 +264 0.104109 30532 +265 0.0617903 30995 +266 0.0338421 31042 +267 0.0174647 30620 +268 0.00867821 29589 +269 0.00419968 28293 +270 0.00203244 26916 +271 0.00100204 25464 +272 0.000555586 24219 +273 0.000555599 23207 +274 0.00055582 22187 +275 0.00055516 21136 +276 0.000555436 20243 +277 0.000555618 19426 +278 0.000556778 18635 +279 0.000556976 17870 +280 0.000557162 17190 +281 0.0007904 16506 +282 0.00154557 15837 +283 0.00302973 15234 +284 0.00584543 14717 +285 0.0108796 14225 +286 0.0192919 13810 +287 0.032329 13605 +288 0.0505293 13536 +289 0.0733417 13760 +290 0.0982413 14378 +291 0.121477 15400 +292 0.138636 17017 +293 0.14875 18764 +294 0.150515 20516 +295 0.146372 22389 +296 0.137332 23975 +297 0.124076 25120 +298 0.106469 26137 +299 0.0862987 26973 +300 0.0650552 27584 +301 0.0456456 27741 +302 0.0300744 27565 +303 0.0187879 27212 +304 0.0112085 26432 +305 0.00648306 25501 +306 0.00370346 24466 +307 0.00213399 23472 +308 0.00125463 22415 +309 0.000765794 21427 +310 0.000552587 20533 +311 0.000553175 19632 +312 0.000553525 18831 +313 0.000554941 18119 +314 0.000556327 17336 +315 0.000556008 16721 +316 0.00055593 16086 +317 0.000556421 15516 +318 0.000557308 14918 +319 0.00060681 14402 +320 0.000990746 13849 +321 0.00172359 13355 +322 0.00313688 12902 +323 0.0058708 12425 +324 0.0110637 12087 +325 0.0206777 11743 +326 0.0376394 11531 +327 0.0656182 11582 +328 0.107414 12034 +329 0.162101 12955 +330 0.223525 14571 +331 0.279935 16842 +332 0.314601 19566 +333 0.313556 22575 +334 0.279571 25279 +335 0.221638 27642 +336 0.158038 29275 +337 0.102505 30638 +338 0.0608328 31209 +339 0.0335531 31260 +340 0.0173332 30520 +341 0.00861545 29604 +342 0.00419454 28370 +343 0.00202587 26940 +344 0.000994029 25614 +345 0.000549339 24445 +346 0.000551477 23239 +347 0.000552891 22300 +348 0.000551775 21280 +349 0.000552425 20424 +350 0.000552135 19571 +351 0.000552542 18753 +352 0.000552863 18058 +353 0.000554438 17348 +354 0.000786735 16671 +355 0.00153958 16047 +356 0.00301482 15500 +357 0.00580589 14883 +358 0.0108227 14347 +359 0.0192357 13947 +360 0.0321613 13672 +361 0.050229 13606 +362 0.0729462 13815 +363 0.0978564 14566 +364 0.120879 15674 +365 0.137663 17049 +366 0.147092 18813 +367 0.150184 20578 +368 0.146971 22245 +369 0.136769 23723 +370 0.12367 24905 +371 0.106187 25871 +372 0.0860921 26687 +373 0.0645899 27375 +374 0.0453473 27635 +375 0.0298122 27551 +376 0.0185448 27134 +377 0.0110517 26468 +378 0.00640294 25661 +379 0.00367011 24653 +380 0.00211832 23556 +381 0.00125246 22513 +382 0.00076891 21568 +383 0.000557384 20672 +384 0.000557295 19811 +385 0.000556837 18982 +386 0.000557433 18179 +387 0.000557376 17457 +388 0.000557751 16720 +389 0.000556844 16112 +390 0.000555603 15479 +391 0.000554871 14809 +392 0.00060335 14275 +393 0.000982808 13757 +394 0.00170757 13221 +395 0.00310351 12758 +396 0.0058181 12286 +397 0.010991 11906 +398 0.0205342 11557 +399 0.0373486 11393 +400 0.0647659 11487 +401 0.105589 11887 +402 0.15967 12798 +403 0.220945 14260 +404 0.277122 16477 +405 0.310108 19295 +406 0.308854 22110 +407 0.274911 24915 +408 0.218618 27273 +409 0.156618 29189 +410 0.101775 30572 +411 0.0607503 31174 +412 0.0334708 31316 +413 0.0173443 30731 +414 0.00865633 29636 +415 0.00421141 28342 +416 0.00204387 26991 +417 0.00100602 25595 +418 0.000555131 24336 +419 0.000555037 23251 +420 0.000555559 22267 +421 0.000554916 21212 +422 0.000554432 20306 +423 0.000554751 19488 +424 0.00055638 18727 +425 0.000556727 17927 +426 0.000556368 17198 +427 0.000788004 16578 +428 0.00154404 15944 +429 0.00302383 15315 +430 0.00582586 14786 +431 0.0108457 14290 +432 0.0192962 13815 +433 0.0323072 13561 +434 0.0505101 13456 +435 0.0732162 13811 +436 0.0978737 14403 +437 0.121405 15460 +438 0.138202 16993 +439 0.1482 18710 +440 0.149707 20578 +441 0.146945 22256 +442 0.137785 23713 +443 0.123767 25058 +444 0.105989 26087 +445 0.085483 26759 +446 0.0646144 27375 +447 0.0454389 27680 +448 0.0299337 27531 +449 0.018663 27041 +450 0.0111347 26416 +451 0.00644197 25614 +452 0.00369229 24666 +453 0.00211986 23647 +454 0.00124761 22650 +455 0.000769104 21642 +456 0.000558796 20693 +457 0.000559908 19746 +458 0.000559562 18952 +459 0.00056042 18100 +460 0.000559447 17401 +461 0.000557893 16756 +462 0.000557137 16148 +463 0.000557269 15504 +464 0.000557596 14974 +465 0.000606298 14408 +466 0.000987712 13909 +467 0.00171257 13402 +468 0.00311667 12891 +469 0.00584794 12433 +470 0.0110774 11980 +471 0.0207006 11713 +472 0.037673 11583 +473 0.0654988 11677 +474 0.106982 12072 +475 0.161926 12898 +476 0.224327 14548 +477 0.281709 16796 +478 0.314567 19512 +479 0.313419 22428 +480 0.278962 25186 +481 0.221864 27755 +482 0.158559 29556 +483 0.103532 30572 +484 0.0611592 31162 +485 0.0337539 31197 +486 0.0175096 30619 +487 0.00865906 29606 +488 0.00420125 28271 +489 0.00203207 26856 +490 0.00100238 25542 +491 0.000554405 24306 +492 0.00055373 23160 +493 0.0005552 22152 +494 0.000553776 21192 +495 0.000553636 20302 +496 0.000553165 19505 +497 0.000554014 18719 +498 0.00055519 17993 +499 0.000556582 17233 +500 0.000788165 16569 +501 0.00154132 15953 +502 0.00302099 15350 +503 0.00581186 14752 +504 0.0108291 14267 +505 0.0192368 13946 +506 0.0322191 13677 +507 0.0503789 13594 +508 0.0730706 13768 +509 0.0980646 14416 +510 0.121601 15634 +511 0.139046 17110 +512 0.147779 18876 +513 0.149612 20734 +514 0.145796 22414 +515 0.136936 23884 +516 0.123807 25078 +517 0.106212 26066 +518 0.0855482 26779 +519 0.0643386 27340 +520 0.0452926 27530 +521 0.0298659 27573 +522 0.0185447 27169 +523 0.0110178 26489 +524 0.00635235 25588 +525 0.00362881 24549 +526 0.00209238 23528 +527 0.00123133 22541 +528 0.000755917 21498 +529 0.000546368 20607 +530 0.000547382 19712 +531 0.000547084 18975 +532 0.000546453 18178 +533 0.000546062 17452 +534 0.000546085 16749 +535 0.000546151 16135 +536 0.000545628 15567 +537 0.000545969 14968 +538 0.000594606 14392 +539 0.000968849 13854 +540 0.00168489 13360 +541 0.00306337 12899 +542 0.00573505 12407 +543 0.0108348 12017 +544 0.02025 11713 +545 0.0368201 11517 +546 0.0639795 11556 +547 0.104882 11941 +548 0.158923 12854 +549 0.219796 14396 +550 0.275801 16733 +551 0.307622 19367 +552 0.30785 22230 +553 0.272898 24873 +554 0.217351 27152 +555 0.156138 29108 +556 0.101477 30379 +557 0.0601091 30971 +558 0.0331551 31126 +559 0.017167 30418 +560 0.00853886 29430 +561 0.00415201 28190 +562 0.00201849 26849 +563 0.000991957 25528 +564 0.000546751 24180 +565 0.00054534 23090 +566 0.000544403 22096 +567 0.00054368 21140 +568 0.000543407 20213 +569 0.000544421 19405 +570 0.000545241 18625 +571 0.000546995 17868 +572 0.000547101 17102 +573 0.00077428 16423 +574 0.00151348 15783 +575 0.00296212 15220 +576 0.00569555 14602 +577 0.0106307 14154 +578 0.0188783 13743 +579 0.0316572 13538 +580 0.0495211 13467 +581 0.0718936 13665 +582 0.0961304 14240 +583 0.119127 15341 +584 0.136233 16912 +585 0.145327 18567 +586 0.146983 20301 +587 0.143022 21953 +588 0.134931 23439 +589 0.121892 24750 +590 0.103955 25688 +591 0.0833804 26253 +592 0.0625106 26918 +593 0.0440419 27279 +594 0.0290823 27159 +595 0.0180758 26786 +596 0.0107654 26049 +597 0.00622673 25202 +598 0.00356716 24168 +599 0.00205866 23122 +600 0.00121254 22076 +601 0.000745744 21100 +602 0.000537789 20207 +603 0.000537982 19340 +604 0.000537795 18527 +605 0.000537955 17768 +606 0.000539259 17117 +607 0.00053942 16425 +608 0.000540477 15701 +609 0.000540424 15134 +610 0.000540084 14558 +611 0.00058571 14069 +612 0.00095364 13498 +613 0.00165505 13054 +614 0.00300205 12616 +615 0.00561724 12142 +616 0.0106079 11720 +617 0.0198178 11410 +618 0.0360368 11231 +619 0.0623418 11314 +620 0.101856 11688 +621 0.15376 12623 +622 0.213046 14078 +623 0.267285 16225 +624 0.299225 18856 +625 0.299517 21756 +626 0.26697 24652 +627 0.2119 27051 +628 0.151393 28925 +629 0.098869 30065 +630 0.0593653 30570 +631 0.0327177 30483 +632 0.0170081 29735 +633 0.0084493 28844 +634 0.00409333 27665 +635 0.00197466 26356 +636 0.000967996 25009 +637 0.000533137 23839 +638 0.000532992 22721 +639 0.000534258 21676 +640 0.000534251 20709 +641 0.000534556 19798 +642 0.000535287 19008 +643 0.000536214 18278 +644 0.000536647 17547 +645 0.000536556 16901 +646 0.000761043 16256 +647 0.00149108 15621 +648 0.00292808 15032 +649 0.0056527 14504 +650 0.0105421 14010 +651 0.0186823 13646 +652 0.0312164 13356 +653 0.0485643 13404 +654 0.0704061 13612 +655 0.0945219 14230 +656 0.117178 15374 +657 0.134568 16843 +658 0.144475 18492 +659 0.146915 20238 +660 0.14393 21958 +661 0.134621 23537 +662 0.121737 24773 +663 0.104744 25772 +664 0.0846226 26427 +665 0.0639754 27040 +666 0.0448457 27279 +667 0.029482 27106 +668 0.0183036 26853 +669 0.0108721 26178 +670 0.00627116 25425 +671 0.0035776 24326 +672 0.00206466 23279 +673 0.00122064 22191 +674 0.000751578 21231 +675 0.000542574 20323 +676 0.000540396 19496 +677 0.000538805 18651 +678 0.00053881 17920 +679 0.000537801 17217 +680 0.000537866 16520 +681 0.000538522 15876 +682 0.000538795 15229 +683 0.000539519 14656 +684 0.000587348 14121 +685 0.000955855 13626 +686 0.00165656 13086 +687 0.00301095 12666 +688 0.00564993 12250 +689 0.0106767 11869 +690 0.0199729 11524 +691 0.03641 11331 +692 0.0632378 11402 +693 0.103483 11788 +694 0.156399 12682 +695 0.215591 14337 +696 0.269462 16547 +697 0.303615 19239 +698 0.304506 22023 +699 0.273068 24769 +700 0.21682 27223 +701 0.154934 29029 +702 0.100495 30241 +703 0.0597382 30801 +704 0.0329221 30881 +705 0.0170591 30288 +706 0.00845353 29329 +707 0.00408176 28108 +708 0.00198037 26715 +709 0.000977102 25340 +710 0.000541566 24039 +711 0.000542333 22965 +712 0.000542417 21858 +713 0.000541182 20952 +714 0.00054038 20049 +715 0.000539725 19192 +716 0.000539603 18409 +717 0.000539754 17700 +718 0.000539679 16960 +719 0.000763508 16287 +720 0.00149327 15637 +721 0.00292609 15057 +722 0.00563308 14524 +723 0.0104893 14003 +724 0.0185874 13625 +725 0.0310985 13319 +726 0.0487417 13278 +727 0.0707124 13502 +728 0.0947795 14147 +729 0.117155 15183 +730 0.133995 16622 diff --git a/third_party/rust/flate2/tests/gunzip.rs b/third_party/rust/flate2/tests/gunzip.rs new file mode 100644 index 0000000000..855c620b0b --- /dev/null +++ b/third_party/rust/flate2/tests/gunzip.rs @@ -0,0 +1,77 @@ +extern crate flate2; + +use flate2::read::GzDecoder; +use flate2::read::MultiGzDecoder; +use std::fs::File; +use std::io::prelude::*; +use std::io::{self, BufReader}; +use std::path::Path; + +// test extraction of a gzipped file +#[test] +fn test_extract_success() { + let content = extract_file(Path::new("tests/good-file.gz")).unwrap(); + let mut expected = Vec::new(); + File::open("tests/good-file.txt") + .unwrap() + .read_to_end(&mut expected) + .unwrap(); + assert!(content == expected); +} +// +// test partial extraction of a multistream gzipped file +#[test] +fn test_extract_success_partial_multi() { + let content = extract_file(Path::new("tests/multi.gz")).unwrap(); + let mut expected = String::new(); + BufReader::new(File::open("tests/multi.txt").unwrap()) + .read_line(&mut expected) + .unwrap(); + assert_eq!(content, expected.as_bytes()); +} + +// test extraction fails on a corrupt file +#[test] +fn test_extract_failure() { + let result = extract_file(Path::new("tests/corrupt-file.gz")); + assert_eq!(result.err().unwrap().kind(), io::ErrorKind::InvalidInput); +} + +//test complete extraction of a multistream gzipped file +#[test] +fn test_extract_success_multi() { + let content = extract_file_multi(Path::new("tests/multi.gz")).unwrap(); + let mut expected = Vec::new(); + File::open("tests/multi.txt") + .unwrap() + .read_to_end(&mut expected) + .unwrap(); + assert_eq!(content, expected); +} + +// Tries to extract path into memory (assuming a .gz file). +fn extract_file(path_compressed: &Path) -> io::Result<Vec<u8>> { + let mut v = Vec::new(); + let f = File::open(path_compressed)?; + GzDecoder::new(f).read_to_end(&mut v)?; + Ok(v) +} + +// Tries to extract path into memory (decompressing all members in case +// of a multi member .gz file). +fn extract_file_multi(path_compressed: &Path) -> io::Result<Vec<u8>> { + let mut v = Vec::new(); + let f = File::open(path_compressed)?; + MultiGzDecoder::new(f).read_to_end(&mut v)?; + Ok(v) +} + +#[test] +fn empty_error_once() { + let data: &[u8] = &[]; + let cbjson = GzDecoder::new(data); + let reader = BufReader::new(cbjson); + let mut stream = reader.lines(); + assert!(stream.next().unwrap().is_err()); + assert!(stream.next().is_none()); +} diff --git a/third_party/rust/flate2/tests/multi.gz b/third_party/rust/flate2/tests/multi.gz Binary files differnew file mode 100644 index 0000000000..cabc89630f --- /dev/null +++ b/third_party/rust/flate2/tests/multi.gz diff --git a/third_party/rust/flate2/tests/multi.txt b/third_party/rust/flate2/tests/multi.txt new file mode 100644 index 0000000000..66a52ee7a1 --- /dev/null +++ b/third_party/rust/flate2/tests/multi.txt @@ -0,0 +1,2 @@ +first +second diff --git a/third_party/rust/flate2/tests/tokio.rs b/third_party/rust/flate2/tests/tokio.rs new file mode 100644 index 0000000000..0f73646407 --- /dev/null +++ b/third_party/rust/flate2/tests/tokio.rs @@ -0,0 +1,133 @@ +#![cfg(feature = "tokio")] + +extern crate flate2; +extern crate futures; +extern crate rand; +extern crate tokio_io; +extern crate tokio_tcp; +extern crate tokio_threadpool; + +use std::io::{Read, Write}; +use std::iter; +use std::net::{Shutdown, TcpListener}; +use std::thread; + +use flate2::read; +use flate2::write; +use flate2::Compression; +use futures::Future; +use rand::{thread_rng, Rng}; +use tokio_io::io::{copy, shutdown}; +use tokio_io::AsyncRead; +use tokio_tcp::TcpStream; + +#[test] +fn tcp_stream_echo_pattern() { + const N: u8 = 16; + const M: usize = 16 * 1024; + + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let t = thread::spawn(move || { + let a = listener.accept().unwrap().0; + let b = a.try_clone().unwrap(); + + let t = thread::spawn(move || { + let mut b = read::DeflateDecoder::new(b); + let mut buf = [0; M]; + for i in 0..N { + b.read_exact(&mut buf).unwrap(); + for byte in buf.iter() { + assert_eq!(*byte, i); + } + } + + assert_eq!(b.read(&mut buf).unwrap(), 0); + }); + + let mut a = write::ZlibEncoder::new(a, Compression::default()); + for i in 0..N { + let buf = [i; M]; + a.write_all(&buf).unwrap(); + } + a.finish().unwrap().shutdown(Shutdown::Write).unwrap(); + + t.join().unwrap(); + }); + + let stream = TcpStream::connect(&addr); + let copy = stream + .and_then(|s| { + let (a, b) = s.split(); + let a = read::ZlibDecoder::new(a); + let b = write::DeflateEncoder::new(b, Compression::default()); + copy(a, b) + }) + .then(|result| { + let (amt, _a, b) = result.unwrap(); + assert_eq!(amt, (N as u64) * (M as u64)); + shutdown(b).map(|_| ()) + }) + .map_err(|err| panic!("{}", err)); + + let threadpool = tokio_threadpool::Builder::new().build(); + threadpool.spawn(copy); + threadpool.shutdown().wait().unwrap(); + t.join().unwrap(); +} + +#[test] +fn echo_random() { + let v = iter::repeat(()) + .take(1024 * 1024) + .map(|()| thread_rng().gen::<u8>()) + .collect::<Vec<_>>(); + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let addr = listener.local_addr().unwrap(); + let v2 = v.clone(); + let t = thread::spawn(move || { + let a = listener.accept().unwrap().0; + let b = a.try_clone().unwrap(); + + let mut v3 = v2.clone(); + let t = thread::spawn(move || { + let mut b = read::DeflateDecoder::new(b); + let mut buf = [0; 1024]; + while v3.len() > 0 { + let n = b.read(&mut buf).unwrap(); + for (actual, expected) in buf[..n].iter().zip(&v3) { + assert_eq!(*actual, *expected); + } + v3.drain(..n); + } + + assert_eq!(b.read(&mut buf).unwrap(), 0); + }); + + let mut a = write::ZlibEncoder::new(a, Compression::default()); + a.write_all(&v2).unwrap(); + a.finish().unwrap().shutdown(Shutdown::Write).unwrap(); + + t.join().unwrap(); + }); + + let stream = TcpStream::connect(&addr); + let copy = stream + .and_then(|s| { + let (a, b) = s.split(); + let a = read::ZlibDecoder::new(a); + let b = write::DeflateEncoder::new(b, Compression::default()); + copy(a, b) + }) + .then(move |result| { + let (amt, _a, b) = result.unwrap(); + assert_eq!(amt, v.len() as u64); + shutdown(b).map(|_| ()) + }) + .map_err(|err| panic!("{}", err)); + + let threadpool = tokio_threadpool::Builder::new().build(); + threadpool.spawn(copy); + threadpool.shutdown().wait().unwrap(); + t.join().unwrap(); +} diff --git a/third_party/rust/flate2/tests/zero-write.rs b/third_party/rust/flate2/tests/zero-write.rs new file mode 100644 index 0000000000..f0db86cb8d --- /dev/null +++ b/third_party/rust/flate2/tests/zero-write.rs @@ -0,0 +1,8 @@ +extern crate flate2; + +#[test] +fn zero_write_is_error() { + let mut buf = [0u8]; + let writer = flate2::write::DeflateEncoder::new(&mut buf[..], flate2::Compression::default()); + assert!(writer.finish().is_err()); +} |