diff options
Diffstat (limited to 'vendor/gix-pack')
30 files changed, 936 insertions, 250 deletions
diff --git a/vendor/gix-pack/.cargo-checksum.json b/vendor/gix-pack/.cargo-checksum.json index 3b289f74a..9fa31bc91 100644 --- a/vendor/gix-pack/.cargo-checksum.json +++ b/vendor/gix-pack/.cargo-checksum.json @@ -1 +1 @@ -{"files":{"CHANGELOG.md":"433166e0184332725634d244c933ee396c06949d5e47c71072ca7ab9c98f8360","Cargo.toml":"34f6280d2d0aaca71e37f13290a77eec51a74f5e6e67e43fc0e3f99d462b01c6","src/bundle/find.rs":"6d8373ae8863e14bcbbcbf2005e43e151cee8e25336a9f79dddd6bdbb85c7b27","src/bundle/init.rs":"6067e968404f52dcd38ed5c6c0114f880e9ff7bd0f883a07b7546e22678aabdc","src/bundle/mod.rs":"e4ee9d88ba45da113f0d9081535a3218bb3a3ebdb51ca7af52fe92fd33021b00","src/bundle/write/error.rs":"f978b1622ac1354f44a7160d2439d5ac697dd92b67631d8c846ae2621be489ae","src/bundle/write/mod.rs":"29330fa5aa340a90f8073aa965319f7186bc96773762a10cd6b754f0a40f2d41","src/bundle/write/types.rs":"bf7db8e54d4c8ddad562185a1a542b8fec5905a047433e37c227bb31e94f630a","src/cache/delta/from_offsets.rs":"721b2f9dc805f073b5a96a8dcf1f27ebc5c2795fe53476eb45116d464bc309b3","src/cache/delta/mod.rs":"d2d67cc96fb8e0fe48bc6eabdc2846c448f8bb297ad0512e0dfc40d54205b3cb","src/cache/delta/traverse/mod.rs":"d3efb10012c2cb73cb570f66b25f69d241cdd989ec3acdebc7d1dc2708331a15","src/cache/delta/traverse/resolve.rs":"dc67edb43d830c32cabad943cc93890fbfbc9ac498ba54fd584a8fc27d0bb973","src/cache/delta/traverse/util.rs":"da1bed028290df6b2730e097404a5590466f59bc4f7de3b1ab24d90e2c6f6692","src/cache/lru.rs":"b4acd8a8757c4d7b23ed00efe2c26cfdd100a54274f2da5ef2e1ffe21aab2194","src/cache/mod.rs":"1fd545e5944ec378f5a89825840fc869684ca571ea7859213ea38d674e991fb1","src/cache/object.rs":"a643dd8418a08c185a2d0c42e18ddaa3809f233b3846c96377e7c92c85492cf7","src/data/delta.rs":"915c10207f5897d473cc806ae31de98e90f41e5e99361e18cb1e42177137c024","src/data/entry/decode.rs":"629abef1cd8352bb57c34ff1b6282e6387a12ec67372b823e0d0dda4caafd901","src/data/entry/header.rs":"55ca90e93c4b39d09b19bfe10dbb74792258cd5f73df29005b60dfc671669024","src/data/entry/mod.rs":"543c2ed9985bdc7077655deaaeb4d1745d03ae74db03842ae97e2c0fb4880db7","src/data/file/decode/entry.rs":"e98ffe4e96594b0ef396708a03d3aefea102fe474136b4c9fae72fb8e7d02346","src/data/file/decode/header.rs":"374bfe93eec705204eb5cb1f3cc97736160f4f05eb629928768af73b058c9361","src/data/file/decode/mod.rs":"bed7398ba3c6a48d2c7af96a2044ee56ba3d9e757995e06493517db27c743712","src/data/file/init.rs":"16a594f4245be493e3ca61fabe730d796e1d2235de7c0b38609e394df9737c86","src/data/file/mod.rs":"4b6a8a99a4864365592733836d654cc30ab717bf8a69569d02fac3ef0b88fac2","src/data/file/verify.rs":"20aea340799f68087aa196be574fe48cff25047cd1dfdaa99b1fb2f60f4319d9","src/data/header.rs":"cc86b9b6d45c7764217bcea615e5640fbbb4af0d17d25cc24073d48d2b9fd206","src/data/input/bytes_to_entries.rs":"a52ebe1de8ea67cacd610d6999d165185b7c2b1c916126ffe7de5df402927a92","src/data/input/entries_to_bytes.rs":"01f23c0cf5822c1f3f5888d287a0f03f1e67dc519d83061ccbca9c5a38adfff0","src/data/input/entry.rs":"9e9d9f2a696e084e71a7a50d85b85003647e4a761f281b3f0623333454d5352a","src/data/input/lookup_ref_delta_objects.rs":"ad122bdfdb07fee85695b2d0909606720917c3ed0ea7f1d1a81de4cab2ce2eac","src/data/input/mod.rs":"386167c1f33fad1b20718c8664a7bd2984c498affaffa29cc54dff13a5d9f3b8","src/data/input/types.rs":"b48be6950d83ebb4019a5a53ced7fa37b6763f4f50510230fce90761ca85d7ae","src/data/mod.rs":"2e65995227c8745b4a62695cf434f0efbf5e136810cf4ac3c1ee09c9e8e210f0","src/data/output/bytes.rs":"85ae6a3e44c569ba842a770a65ce549dbf772d29fa55368f263c0cae3447443e","src/data/output/count/mod.rs":"16d2a4a502d39ebeaee809c5570089288c9d470613ef483b12393c5b0ae0fd32","src/data/output/count/objects/mod.rs":"b04d5360c9c37643b17199b2cb89faa5d576d7cabe9d8ea9ab2c5f0ad217aead","src/data/output/count/objects/reduce.rs":"22371344975483bfd8b3a3dec67cd290a1cb526967b0c52032f817bcdba96014","src/data/output/count/objects/tree.rs":"7d6bfbe55d32c3c13fe1c9004e0671a8fc7379d73985ef69da0d1d2d153315e3","src/data/output/count/objects/types.rs":"7665bda20fdb05673f71b57d4e838e3327d2a151ab20e0bc7635ef6c203abb43","src/data/output/count/objects/util.rs":"f018375fa5d1dbf48a714f99768b6ab45e5403a2ed59b70829dae172c65c187b","src/data/output/entry/iter_from_counts.rs":"58c9cc17b7ea728b781c4f30a0a5ebc72601ccf1cc4271492584bfce015db032","src/data/output/entry/mod.rs":"d1b9b4480bdfb1821c3a3153a5f3c880584a2f0a46a65a187ce676406b6b03fa","src/data/output/mod.rs":"ac037fc6111d8f32f1d807bb3d34c2af351b99bdc97c600992d64c90fdb4fa1e","src/find.rs":"9a5025b69728b6ce2358a9234bee46f4d20dcc2dcd0925faaada8c71d0c13741","src/find_traits.rs":"04cf9445ff46a29cb4f9c91db846cf4288b341671d60f3362bdd0e4f36c87a01","src/index/access.rs":"1f8080929954ac3fe312ae7ac94ae9e3f33563370ff76742e748008145fc0a75","src/index/init.rs":"d25b0865859f505fba4b02437faad6f02b264585906c9cdc1743b64f7f238148","src/index/mod.rs":"5b4539665da73770f76628863ec3c1d4532f6033cd272a6c0fb7ea88856d2570","src/index/traverse/error.rs":"d520a384e3737ac973a9d84cf6dbd7ebda6f459d26560e6e40555eacede8c7f6","src/index/traverse/mod.rs":"cb6392c1182af69de5e114804fcda9f964160028b8e6346b5b3375028328b5c9","src/index/traverse/reduce.rs":"0f3614232888f66c5ad13552875ced211c79dad656093e080b16bfc25ff5d7b1","src/index/traverse/types.rs":"a9ad1e3b6944d62c38fcde4823a46809736516e9753dc16d15f38ac2bb74041e","src/index/traverse/with_index.rs":"5bd064a735bffca6a20708998a78d9e75ea75947d6bc6b8c318d00854eb80ece","src/index/traverse/with_lookup.rs":"013aca8001fa4e019bf071c5b7ce2e1b2c274bd055f7a1705d367e72fa68b39c","src/index/util.rs":"546454f39d469b2b1cca384822e3004a48b9c6a91b899cce83b5759026911419","src/index/verify.rs":"736450fa7b3164dddbbfa0364a8c5cf62297f3cfb622c6e0097288bdc01cafd1","src/index/write/encode.rs":"d92c85fd80ff380470f280720ddfe19422671ad24d6444c9f3a4864608efcd91","src/index/write/error.rs":"5294efe0508252d5942183fa5ab5139dc10e33ccfb28698a6101fc360990d688","src/index/write/mod.rs":"2ed22637098ce8b70c3c7321f7cd2bb9e539ad70f5e72be2a6255d41c6bd9153","src/lib.rs":"7b72df6596c2501d8bb9c0bde09620e6b497ce4561191c59eae0d4229356d97b","src/multi_index/access.rs":"5c6b309147423c27defc2f7fbd194a4c2d60e7c9d9f77b2fccfcb647ef426140","src/multi_index/chunk.rs":"c3e2e08f485db6043c8ae74a1d3daab6f619ba17cdc0af32c5d010ec433f97d2","src/multi_index/init.rs":"290daf86cfe21127a27c5bea49d3b1ef4812bde968ff30b36e4cef278bc513c9","src/multi_index/mod.rs":"38ac324b03c7ed2a7a3d169ff1997a7799f343c86bf9b5b026be00def7685fc9","src/multi_index/verify.rs":"b165566234f53abde696b741843000f55a5699c90d38e03173fa6f58279c4b3f","src/multi_index/write.rs":"caa956a4a5c504226492367facb0acd5f5ba410f9e4f7e86c7b668a5605998f7","src/verify.rs":"5e5d9decdbfb46963b5589dd49d76079e28a8aa6575d20d078492a7f2d50bad9"},"package":"164a515900a83257ae4aa80e741655bee7a2e39113fb535d7a5ac623b445ff20"}
\ No newline at end of file +{"files":{"CHANGELOG.md":"03315402ee906c24a2e0c846a7246c36fe29f37119f447f149d6c4d5030e54f1","Cargo.toml":"75f30d0efeddf596a1145e39ad38a21a3b207b28ce062691e7edfedcf7aae447","LICENSE-APACHE":"cb4780590812826851ba250f90bed0ed19506ec98f6865a0e2e20bbf62391ff9","LICENSE-MIT":"49df47913ab2beafe8dc45607877ae64198bf0eee64aaad3e82ed9e4d27424e8","src/bundle/find.rs":"f8b2fb56e6b898b7da15cfb278bf943641244eb0e5a630d56b7146eb7029911f","src/bundle/init.rs":"6067e968404f52dcd38ed5c6c0114f880e9ff7bd0f883a07b7546e22678aabdc","src/bundle/mod.rs":"e4ee9d88ba45da113f0d9081535a3218bb3a3ebdb51ca7af52fe92fd33021b00","src/bundle/write/error.rs":"f978b1622ac1354f44a7160d2439d5ac697dd92b67631d8c846ae2621be489ae","src/bundle/write/mod.rs":"1d191d9420a9f0fc59918fbf451521677da94680357dcb78bb61f4cd05dfdb0b","src/bundle/write/types.rs":"faef010d8680b85b08ad2aaca291fed1e07ddf00a5f407f1c60d657b0ef6ed5b","src/cache/delta/from_offsets.rs":"7e85dc125810ab0c01a2aac0f846b540377e0f47fe62d31c2277410264fb6468","src/cache/delta/mod.rs":"d2d67cc96fb8e0fe48bc6eabdc2846c448f8bb297ad0512e0dfc40d54205b3cb","src/cache/delta/traverse/mod.rs":"dff2a3fe900e9f8b5dd5d49e461e48eed7d73331eaa22bdc199ee6faa20fd599","src/cache/delta/traverse/resolve.rs":"0748b96962ba415679808597a9e0cc0a2552a756e5785895c5903378d4bb6690","src/cache/delta/traverse/util.rs":"eced32d724ad66766b5c9933d0324ef351436740266a85491238ec8d26010b7d","src/cache/lru.rs":"557043f806f754fadae96199d2261a8df184f86bf0dd525efe091c5fa3a793da","src/cache/mod.rs":"1fd545e5944ec378f5a89825840fc869684ca571ea7859213ea38d674e991fb1","src/cache/object.rs":"5b796f4b1776290a56f258a99132d2aad2fd65851de0c2476fb7766a2325931e","src/data/delta.rs":"915c10207f5897d473cc806ae31de98e90f41e5e99361e18cb1e42177137c024","src/data/entry/decode.rs":"629abef1cd8352bb57c34ff1b6282e6387a12ec67372b823e0d0dda4caafd901","src/data/entry/header.rs":"55ca90e93c4b39d09b19bfe10dbb74792258cd5f73df29005b60dfc671669024","src/data/entry/mod.rs":"543c2ed9985bdc7077655deaaeb4d1745d03ae74db03842ae97e2c0fb4880db7","src/data/file/decode/entry.rs":"21a52cd2e12c8464e1b53f92a8c4724215ab90aacb29ec6ca7a98da74048c309","src/data/file/decode/header.rs":"374bfe93eec705204eb5cb1f3cc97736160f4f05eb629928768af73b058c9361","src/data/file/decode/mod.rs":"bed7398ba3c6a48d2c7af96a2044ee56ba3d9e757995e06493517db27c743712","src/data/file/init.rs":"16a594f4245be493e3ca61fabe730d796e1d2235de7c0b38609e394df9737c86","src/data/file/mod.rs":"4b6a8a99a4864365592733836d654cc30ab717bf8a69569d02fac3ef0b88fac2","src/data/file/verify.rs":"20aea340799f68087aa196be574fe48cff25047cd1dfdaa99b1fb2f60f4319d9","src/data/header.rs":"cc86b9b6d45c7764217bcea615e5640fbbb4af0d17d25cc24073d48d2b9fd206","src/data/input/bytes_to_entries.rs":"d60b835cb1c4e25b98faac8c1e48d1830352fca81e1b019b6c7ee784b8ab4f7b","src/data/input/entries_to_bytes.rs":"01f23c0cf5822c1f3f5888d287a0f03f1e67dc519d83061ccbca9c5a38adfff0","src/data/input/entry.rs":"9e9d9f2a696e084e71a7a50d85b85003647e4a761f281b3f0623333454d5352a","src/data/input/lookup_ref_delta_objects.rs":"f410de41c7031c5d65b7300ef61f44555b114acf0887432f2f25c64ea320749a","src/data/input/mod.rs":"386167c1f33fad1b20718c8664a7bd2984c498affaffa29cc54dff13a5d9f3b8","src/data/input/types.rs":"b48be6950d83ebb4019a5a53ced7fa37b6763f4f50510230fce90761ca85d7ae","src/data/mod.rs":"2e65995227c8745b4a62695cf434f0efbf5e136810cf4ac3c1ee09c9e8e210f0","src/data/output/bytes.rs":"3fa26f3197bc8cee5fb141cd67c3f233481350422f9acfcb3138f97d58792238","src/data/output/count/mod.rs":"c836960b9952d3982cf575cfbf99035d28625d9802e397906ecc8e0935df0f8d","src/data/output/count/objects/mod.rs":"0477195322c61b65477602e2a29e38c696d7ab6c15c5ea22412dac55c9d6c693","src/data/output/count/objects/reduce.rs":"22371344975483bfd8b3a3dec67cd290a1cb526967b0c52032f817bcdba96014","src/data/output/count/objects/tree.rs":"7d6bfbe55d32c3c13fe1c9004e0671a8fc7379d73985ef69da0d1d2d153315e3","src/data/output/count/objects/types.rs":"b53d36dae89d094d66c622f0f816c412f6401126f7e940caf34856b48e776a4e","src/data/output/count/objects/util.rs":"f018375fa5d1dbf48a714f99768b6ab45e5403a2ed59b70829dae172c65c187b","src/data/output/entry/iter_from_counts.rs":"58c9cc17b7ea728b781c4f30a0a5ebc72601ccf1cc4271492584bfce015db032","src/data/output/entry/mod.rs":"d1b9b4480bdfb1821c3a3153a5f3c880584a2f0a46a65a187ce676406b6b03fa","src/data/output/mod.rs":"ac037fc6111d8f32f1d807bb3d34c2af351b99bdc97c600992d64c90fdb4fa1e","src/find.rs":"9a5025b69728b6ce2358a9234bee46f4d20dcc2dcd0925faaada8c71d0c13741","src/find_traits.rs":"04cf9445ff46a29cb4f9c91db846cf4288b341671d60f3362bdd0e4f36c87a01","src/index/access.rs":"7725c5c810ebc9d93f019913829b02920bbfa1db730e04ad66381280463f63d8","src/index/init.rs":"d25b0865859f505fba4b02437faad6f02b264585906c9cdc1743b64f7f238148","src/index/mod.rs":"5b4539665da73770f76628863ec3c1d4532f6033cd272a6c0fb7ea88856d2570","src/index/traverse/error.rs":"d520a384e3737ac973a9d84cf6dbd7ebda6f459d26560e6e40555eacede8c7f6","src/index/traverse/mod.rs":"c40f4e7f7d65054b6bc7c48769b9a6bae00b8710732cf45a7fa36c445a248509","src/index/traverse/reduce.rs":"0f3614232888f66c5ad13552875ced211c79dad656093e080b16bfc25ff5d7b1","src/index/traverse/types.rs":"a9ad1e3b6944d62c38fcde4823a46809736516e9753dc16d15f38ac2bb74041e","src/index/traverse/with_index.rs":"d9c8d4deecdc56a76deee808981e3e12d7ad37c23e327a9ca5f5b777f4da18b8","src/index/traverse/with_lookup.rs":"39a14d6f0e8235730d489365ff1acf784fad99a779326bab1215229f03b8e2e5","src/index/util.rs":"546454f39d469b2b1cca384822e3004a48b9c6a91b899cce83b5759026911419","src/index/verify.rs":"42ec54de729f176e22bb891950d0ce0e96880730b22e3bcb2a57e0e37f4c2b0e","src/index/write/encode.rs":"250c29516aad527156951280c8fbe263f6c0da4d98899a88a20f8e0187fd11f8","src/index/write/error.rs":"5294efe0508252d5942183fa5ab5139dc10e33ccfb28698a6101fc360990d688","src/index/write/mod.rs":"471f5a1a91b3d4cc6856675f96933eaf5781b3c79b23aaadb01925bdd8b239fa","src/lib.rs":"7b72df6596c2501d8bb9c0bde09620e6b497ce4561191c59eae0d4229356d97b","src/multi_index/access.rs":"5c6b309147423c27defc2f7fbd194a4c2d60e7c9d9f77b2fccfcb647ef426140","src/multi_index/chunk.rs":"7f5506391c4ed94edb83844de67bdbf98a12b28f1ae4e0947e6ad2f6488e3fa3","src/multi_index/init.rs":"290daf86cfe21127a27c5bea49d3b1ef4812bde968ff30b36e4cef278bc513c9","src/multi_index/mod.rs":"38ac324b03c7ed2a7a3d169ff1997a7799f343c86bf9b5b026be00def7685fc9","src/multi_index/verify.rs":"b165566234f53abde696b741843000f55a5699c90d38e03173fa6f58279c4b3f","src/multi_index/write.rs":"133044bad102aceaf8d1ab2e2af239b6466a74011904a74cefec547942b71f1f","src/verify.rs":"5e5d9decdbfb46963b5589dd49d76079e28a8aa6575d20d078492a7f2d50bad9"},"package":"7d2a14cb3156037eedb17d6cb7209b7180522b8949b21fd0fe3184c0a1d0af88"}
\ No newline at end of file diff --git a/vendor/gix-pack/CHANGELOG.md b/vendor/gix-pack/CHANGELOG.md index 70d789808..dad7f017f 100644 --- a/vendor/gix-pack/CHANGELOG.md +++ b/vendor/gix-pack/CHANGELOG.md @@ -5,6 +5,115 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## 0.36.0 (2023-06-06) + +### New Features + + - <csr-id-3a2d5286084597d4c68549903709cda77dda4357/> improve performance by reducing the trashing of `zlib::Inflate`. + Previously, for every zlib inflate operation, we would allocate memory + due to the creation of a decompressor. + + This is now avoided by reusing thread-local copies of such a decompressor, + which is reused and reset instead. + - <csr-id-14e7ea0217af8a04ed2b50ff7b13c28335c29022/> improve performance by avoiding zeroying buffers. + Previously we would use `resize(new_len, 0)` to resize buffers, even though + these values would then be overwritten (or the buffer isn't available). + + Now we use `set_len(new_len)` after calling `reserve` to do the same, but safe + a memset. + +### Bug Fixes + + - <csr-id-977e135bfa00bdbc1a8f8324f85347ec9078c84f/> static linked list delta cache with memory cap + Previously, the 64 slot big LRU cache for pack deltas didn't + use any memory limit which could lead to memory exhaustion in the + face of untypical, large objects. + + Now we add a generous default limit to do *better* in such situations. + It's worth noting though that that even without any cache, the working + set of buffers to do delta resolution takes considerable memory, despite + trying to keep it minimal. + + Note that for bigger objects, the cache is now not used at all, which probably + leads to terrible performance as not even the base object can be cached. + - <csr-id-f89cbc675b0acc67322e289e7b3a17288b9eae27/> check for interrupt more often + Previously when traversing a pack it could appear to hang as checks + were only performed on chunk or base (of a delta-tree) level. + + Now interrupt checks are performed more often to stop all work much quicker. + - <csr-id-923692bcda698b45d3d1ad518b29f6d30b23fbc0/> memory capped hashmap as pack delta cache won't trash memory as much. + Previously it would take a buffer from the free-list, copy data into it, and + when exceeding the capacity loose it entirely. + Now the freelist is handled correctly. + +### New Features (BREAKING) + + - <csr-id-d22dd8fcc22e8dbe30524a1bdddc09bc841db341/> index-backed tree traversal with a form of work-stealing. + When delta-trees are unbalanced, in pathological cases it's possible that that one thread + ends up with more than half of the work. In this case it's required that it manages to + spawn its own threads to parallelize the work it has. + - <csr-id-0fa04bcbdf3102c5435e64cfef894a1bfc8d6e7b/> make current thread-count accessible in slice-workers. + Threads started for working on an entry in a slice can now see the amount + of threads left for use (and manipulate that variable) which effectively + allows them to implement their own parallelization on top of the current one. + + This is useful if there is there is very imbalanced work within the slice itself. + + While at it, we also make consumer functions mutable as they exsit per thread. + +### Commit Statistics + +<csr-read-only-do-not-edit/> + + - 28 commits contributed to the release over the course of 25 calendar days. + - 40 days passed between releases. + - 7 commits were understood as [conventional](https://www.conventionalcommits.org). + - 1 unique issue was worked on: [#851](https://github.com/Byron/gitoxide/issues/851) + +### Thanks Clippy + +<csr-read-only-do-not-edit/> + +[Clippy](https://github.com/rust-lang/rust-clippy) helped 1 time to make code idiomatic. + +### Commit Details + +<csr-read-only-do-not-edit/> + +<details><summary>view details</summary> + + * **[#851](https://github.com/Byron/gitoxide/issues/851)** + - Fix performance regression in pack traversal ([`de1f6c2`](https://github.com/Byron/gitoxide/commit/de1f6c22dfccf22f34e2c2c1b4890ca095ced7ac)) + - Index-backed tree traversal with a form of work-stealing. ([`d22dd8f`](https://github.com/Byron/gitoxide/commit/d22dd8fcc22e8dbe30524a1bdddc09bc841db341)) + - Improve performance by avoiding zeroying buffers. ([`14e7ea0`](https://github.com/Byron/gitoxide/commit/14e7ea0217af8a04ed2b50ff7b13c28335c29022)) + - Make current thread-count accessible in slice-workers. ([`0fa04bc`](https://github.com/Byron/gitoxide/commit/0fa04bcbdf3102c5435e64cfef894a1bfc8d6e7b)) + - Try to get improved hitrate for delta-cache by using it more, and fail ([`969cc77`](https://github.com/Byron/gitoxide/commit/969cc77ec7855fc8c23c2b50353813e6a04b779d)) + - Static linked list delta cache with memory cap ([`977e135`](https://github.com/Byron/gitoxide/commit/977e135bfa00bdbc1a8f8324f85347ec9078c84f)) + - Check for interrupt more often ([`f89cbc6`](https://github.com/Byron/gitoxide/commit/f89cbc675b0acc67322e289e7b3a17288b9eae27)) + - Memory capped hashmap as pack delta cache won't trash memory as much. ([`923692b`](https://github.com/Byron/gitoxide/commit/923692bcda698b45d3d1ad518b29f6d30b23fbc0)) + * **Uncategorized** + - `just fmt` ([`ffc1276`](https://github.com/Byron/gitoxide/commit/ffc1276e0c991ac33ce842f5dca0b45ac69680c0)) + - Prepare changelogs prior to release ([`8f15cec`](https://github.com/Byron/gitoxide/commit/8f15cec1ec7d5a9d56bb158f155011ef2bb3539b)) + - Merge branch 'integrate-gix-negotiate' ([`ae845de`](https://github.com/Byron/gitoxide/commit/ae845dea6cee6523c88a23d7a14293589cf8092f)) + - Thanks clippy ([`9525ac8`](https://github.com/Byron/gitoxide/commit/9525ac822aa902f5325f17e7b08ffb60b683e0e7)) + - Merge branch 'fix-alloc' ([`d9d9bc0`](https://github.com/Byron/gitoxide/commit/d9d9bc01b34ac75b28a5f1b75f40123aa6d83c60)) + - Improve performance by reducing the trashing of `zlib::Inflate`. ([`3a2d528`](https://github.com/Byron/gitoxide/commit/3a2d5286084597d4c68549903709cda77dda4357)) + - Merge branch 'fix-docs' ([`420553a`](https://github.com/Byron/gitoxide/commit/420553a10d780e0b2dc466cac120989298a5f187)) + - Minor fixes ([`89a8cfe`](https://github.com/Byron/gitoxide/commit/89a8cfe40e5c3a9d4a4181fa055e3f4a529a8081)) + - Cleaning up documentation ([`2578e57`](https://github.com/Byron/gitoxide/commit/2578e576bfa365d194a23a1fb0bf09be230873de)) + - Merge branch 'auto-clippy' ([`dbf8aa1`](https://github.com/Byron/gitoxide/commit/dbf8aa19d19109195d0274928eae4b94f248cd88)) + - Autofix map-or-unwrap clippy lint (and manual fix what was left) ([`2087032`](https://github.com/Byron/gitoxide/commit/2087032b5956dcd82bce6ac57e530e8724b57f17)) + - Merge branch 'main' into auto-clippy ([`3ef5c90`](https://github.com/Byron/gitoxide/commit/3ef5c90aebce23385815f1df674c1d28d58b4b0d)) + - Auto-fix clippy to remove explicit iter looping ([`3eff567`](https://github.com/Byron/gitoxide/commit/3eff567c683b5c650c14792b68968cbdbc90ec5c)) + - Merge branch 'blinxen/main' ([`9375cd7`](https://github.com/Byron/gitoxide/commit/9375cd75b01aa22a0e2eed6305fe45fabfd6c1ac)) + - Include license files in all crates ([`facaaf6`](https://github.com/Byron/gitoxide/commit/facaaf633f01c857dcf2572c6dbe0a92b7105c1c)) + - Further optimize buffer usage when traversing packs ([`78d28a7`](https://github.com/Byron/gitoxide/commit/78d28a72a9ed4773b86b36b23bf200269406384e)) + - Fix incorrect naming of progress of traversal threads ([`996ba6b`](https://github.com/Byron/gitoxide/commit/996ba6b20a0e13cd6f448b41797deee97922a3e4)) + - Merge branch 'fix-851' ([`2f275d5`](https://github.com/Byron/gitoxide/commit/2f275d5d3cb49b3b8ba53b30e4b4386fac32662b)) + - Adjust to changes in `gix-features` ([`c48bbe3`](https://github.com/Byron/gitoxide/commit/c48bbe330e5e99fa357a87a4aa210317ab7c8143)) + - Release gix-object v0.29.2 ([`4f879bf`](https://github.com/Byron/gitoxide/commit/4f879bf35653bdc8f9729d524c6e8e1fb3c6886b)) +</details> + ## 0.35.0 (2023-04-27) A maintenance release without user-facing changes. @@ -13,7 +122,7 @@ A maintenance release without user-facing changes. <csr-read-only-do-not-edit/> - - 2 commits contributed to the release. + - 3 commits contributed to the release. - 0 commits were understood as [conventional](https://www.conventionalcommits.org). - 0 issues like '(#ID)' were seen in commit messages @@ -24,6 +133,7 @@ A maintenance release without user-facing changes. <details><summary>view details</summary> * **Uncategorized** + - Release gix-path v0.8.0, gix-glob v0.7.0, gix-attributes v0.12.0, gix-config-value v0.12.0, gix-ref v0.29.0, gix-sec v0.8.0, gix-config v0.22.0, gix-prompt v0.5.0, gix-url v0.18.0, gix-credentials v0.14.0, gix-discover v0.18.0, gix-ignore v0.2.0, gix-pack v0.35.0, gix-odb v0.45.0, gix-transport v0.31.0, gix-protocol v0.32.0, gix-refspec v0.10.1, gix-worktree v0.17.0, gix v0.44.1 ([`7ebc9f7`](https://github.com/Byron/gitoxide/commit/7ebc9f734ec4371dd27daa568c0244185bb49eb5)) - Prepare changelogs prior to release ([`0135158`](https://github.com/Byron/gitoxide/commit/013515897215400539bfd53c25548bd054186ba6)) - Bump gix-path v0.8.0, safety bump 20 crates (gix set to 0.44.1 manually) ([`43ebaf2`](https://github.com/Byron/gitoxide/commit/43ebaf267557218865862538ffc7bdf00558492f)) </details> diff --git a/vendor/gix-pack/Cargo.toml b/vendor/gix-pack/Cargo.toml index 2d2de8bb9..783b411ce 100644 --- a/vendor/gix-pack/Cargo.toml +++ b/vendor/gix-pack/Cargo.toml @@ -10,20 +10,20 @@ # See Cargo.toml.orig for the original contents. [package] -edition = "2018" +edition = "2021" rust-version = "1.64" name = "gix-pack" -version = "0.35.0" +version = "0.36.0" authors = ["Sebastian Thiel <sebastian.thiel@icloud.com>"] include = [ "src/**/*", + "LICENSE-*", "CHANGELOG.md", ] autotests = false description = "Implements git packs and related data structures" license = "MIT/Apache-2.0" repository = "https://github.com/Byron/gitoxide" -resolver = "2" [package.metadata.docs.rs] all-features = true @@ -50,13 +50,13 @@ version = "0.2.0" optional = true [dependencies.gix-chunk] -version = "^0.4.1" +version = "^0.4.2" [dependencies.gix-diff] -version = "^0.29.0" +version = "^0.30.0" [dependencies.gix-features] -version = "^0.29.0" +version = "^0.30.0" features = [ "crc32", "rustsha1", @@ -65,19 +65,19 @@ features = [ ] [dependencies.gix-hash] -version = "^0.11.1" +version = "^0.11.2" [dependencies.gix-hashtable] -version = "^0.2.0" +version = "^0.2.1" [dependencies.gix-object] -version = "^0.29.1" +version = "^0.30.0" [dependencies.gix-path] -version = "^0.8.0" +version = "^0.8.1" [dependencies.gix-traverse] -version = "^0.25.0" +version = "^0.26.0" [dependencies.memmap2] version = "0.5.0" @@ -115,5 +115,5 @@ serde = [ wasm = ["gix-diff/wasm"] [target."cfg(not(target_arch = \"wasm32\"))".dependencies.gix-tempfile] -version = "^5.0.0" +version = "^6.0.0" default-features = false diff --git a/vendor/gix-pack/LICENSE-APACHE b/vendor/gix-pack/LICENSE-APACHE new file mode 100644 index 000000000..a51f59a06 --- /dev/null +++ b/vendor/gix-pack/LICENSE-APACHE @@ -0,0 +1,191 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2018-2021 Sebastian Thiel, and [contributors](https://github.com/byron/gitoxide/contributors) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/gix-pack/LICENSE-MIT b/vendor/gix-pack/LICENSE-MIT new file mode 100644 index 000000000..b58e818f1 --- /dev/null +++ b/vendor/gix-pack/LICENSE-MIT @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018-2021 Sebastian Thiel, and [contributors](https://github.com/byron/gitoxide/contributors). + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/gix-pack/src/bundle/find.rs b/vendor/gix-pack/src/bundle/find.rs index d39ed49a9..2fc335721 100644 --- a/vendor/gix-pack/src/bundle/find.rs +++ b/vendor/gix-pack/src/bundle/find.rs @@ -19,7 +19,7 @@ impl crate::Bundle { } /// Special-use function to get an object given an index previously returned from - /// internal_find_pack_index. + /// `internal_find_pack_index`. /// /// # Panics /// diff --git a/vendor/gix-pack/src/bundle/write/mod.rs b/vendor/gix-pack/src/bundle/write/mod.rs index fc0284b53..103a0034b 100644 --- a/vendor/gix-pack/src/bundle/write/mod.rs +++ b/vendor/gix-pack/src/bundle/write/mod.rs @@ -62,17 +62,14 @@ impl crate::Bundle { /// * the resulting pack may be empty, that is, contains zero objects in some situations. This is a valid reply by a server and should /// be accounted for. /// - Empty packs always have the same name and not handling this case will result in at most one superfluous pack. - pub fn write_to_directory<P>( + pub fn write_to_directory( pack: impl io::BufRead, directory: Option<impl AsRef<Path>>, - mut progress: P, + mut progress: impl Progress, should_interrupt: &AtomicBool, thin_pack_base_object_lookup_fn: Option<ThinPackLookupFn>, options: Options, - ) -> Result<Outcome, Error> - where - P: Progress, - { + ) -> Result<Outcome, Error> { let mut read_progress = progress.add_child_with_id("read pack", ProgressId::ReadPackBytes.into()); read_progress.init(None, progress::bytes()); let pack = progress::Read { @@ -171,7 +168,7 @@ impl crate::Bundle { /// # Note /// /// As it sends portions of the input to a thread it requires the 'static lifetime for the interrupt flags. This can only - /// be satisfied by a static AtomicBool which is only suitable for programs that only run one of these operations at a time + /// be satisfied by a static `AtomicBool` which is only suitable for programs that only run one of these operations at a time /// or don't mind that all of them abort when the flag is set. pub fn write_to_directory_eagerly<P>( pack: impl io::Read + Send + 'static, @@ -354,20 +351,20 @@ impl crate::Bundle { } } +fn resolve_entry(range: data::EntryRange, mapped_file: &memmap2::Mmap) -> Option<&[u8]> { + mapped_file.get(range.start as usize..range.end as usize) +} + fn new_pack_file_resolver( data_file: SharedTempFile, -) -> io::Result<impl Fn(data::EntryRange, &mut Vec<u8>) -> Option<()> + Send + Clone> { +) -> io::Result<( + impl Fn(data::EntryRange, &memmap2::Mmap) -> Option<&[u8]> + Send + Clone, + memmap2::Mmap, +)> { let mut guard = data_file.lock(); guard.flush()?; - let mapped_file = Arc::new(crate::mmap::read_only( - &guard.get_mut().with_mut(|f| f.path().to_owned())?, - )?); - let pack_data_lookup = move |range: std::ops::Range<u64>, out: &mut Vec<u8>| -> Option<()> { - mapped_file - .get(range.start as usize..range.end as usize) - .map(|pack_entry| out.copy_from_slice(pack_entry)) - }; - Ok(pack_data_lookup) + let mapped_file = crate::mmap::read_only(&guard.get_mut().with_mut(|f| f.path().to_owned())?)?; + Ok((resolve_entry, mapped_file)) } struct WriteOutcome { diff --git a/vendor/gix-pack/src/bundle/write/types.rs b/vendor/gix-pack/src/bundle/write/types.rs index 7bc4ec35f..9ade3ed0a 100644 --- a/vendor/gix-pack/src/bundle/write/types.rs +++ b/vendor/gix-pack/src/bundle/write/types.rs @@ -2,8 +2,8 @@ use std::{hash::Hash, io, io::SeekFrom, path::PathBuf, sync::Arc}; use gix_tempfile::handle::Writable; -/// Configuration for [write_to_directory][crate::Bundle::write_to_directory()] or -/// [write_to_directory_eagerly][crate::Bundle::write_to_directory_eagerly()] +/// Configuration for [`write_to_directory`][crate::Bundle::write_to_directory()] or +/// [`write_to_directory_eagerly`][crate::Bundle::write_to_directory_eagerly()] #[derive(Debug, Clone)] pub struct Options { /// The amount of threads to use at most when resolving the pack. If `None`, all logical cores are used. @@ -28,8 +28,8 @@ impl Default for Options { } } -/// Returned by [write_to_directory][crate::Bundle::write_to_directory()] or -/// [write_to_directory_eagerly][crate::Bundle::write_to_directory_eagerly()] +/// Returned by [`write_to_directory`][crate::Bundle::write_to_directory()] or +/// [`write_to_directory_eagerly`][crate::Bundle::write_to_directory_eagerly()] #[derive(PartialEq, Eq, Debug, Hash, Ord, PartialOrd, Clone)] #[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))] pub struct Outcome { diff --git a/vendor/gix-pack/src/cache/delta/from_offsets.rs b/vendor/gix-pack/src/cache/delta/from_offsets.rs index 8acb4a802..065b1ca20 100644 --- a/vendor/gix-pack/src/cache/delta/from_offsets.rs +++ b/vendor/gix-pack/src/cache/delta/from_offsets.rs @@ -31,7 +31,7 @@ const PACK_HEADER_LEN: usize = 12; /// Generate tree from certain input impl<T> Tree<T> { /// Create a new `Tree` from any data sorted by offset, ascending as returned by the `data_sorted_by_offsets` iterator. - /// * `get_pack_offset(item: &T`) -> data::Offset` is a function returning the pack offset of the given item, which can be used + /// * `get_pack_offset(item: &T) -> data::Offset` is a function returning the pack offset of the given item, which can be used /// for obtaining the objects entry within the pack. /// * `pack_path` is the path to the pack file itself and from which to read the entry data, which is a pack file matching the offsets /// returned by `get_pack_offset(…)`. diff --git a/vendor/gix-pack/src/cache/delta/traverse/mod.rs b/vendor/gix-pack/src/cache/delta/traverse/mod.rs index bfe2ec687..e933af838 100644 --- a/vendor/gix-pack/src/cache/delta/traverse/mod.rs +++ b/vendor/gix-pack/src/cache/delta/traverse/mod.rs @@ -3,7 +3,8 @@ use std::sync::atomic::{AtomicBool, Ordering}; use gix_features::{ parallel::in_parallel_with_slice, progress::{self, Progress}, - threading::{lock, Mutable, OwnShared}, + threading, + threading::{Mutable, OwnShared}, }; use crate::{ @@ -36,18 +37,18 @@ pub enum Error { /// The base's offset which was from a resolved ref-delta that didn't actually get added to the tree base_pack_offset: crate::data::Offset, }, + #[error("Failed to spawn thread when switching to work-stealing mode")] + SpawnThread(#[from] std::io::Error), } /// Additional context passed to the `inspect_object(…)` function of the [`Tree::traverse()`] method. -pub struct Context<'a, S> { +pub struct Context<'a> { /// The pack entry describing the object pub entry: &'a crate::data::Entry, /// The offset at which `entry` ends in the pack, useful to learn about the exact range of `entry` within the pack. pub entry_end: u64, /// The decompressed object itself, ready to be decoded. pub decompressed: &'a [u8], - /// Custom state known to the function - pub state: &'a mut S, /// The depth at which this object resides in the delta-tree. It represents the amount of base objects, with 0 indicating /// an 'undeltified' object, and higher values indicating delta objects with the given amount of bases. pub level: u16, @@ -89,76 +90,85 @@ where /// operation as well. /// * `pack_entries_end` marks one-past-the-last byte of the last entry in the pack, as the last entries size would otherwise /// be unknown as it's not part of the index file. - /// * `new_thread_state() -> State` is a function to create state to be used in each thread, invoked once per thread. /// * `inspect_object(node_data: &mut T, progress: Progress, context: Context<ThreadLocal State>) -> Result<(), CustomError>` is a function /// running for each thread receiving fully decoded objects along with contextual information, which either succeeds with `Ok(())` /// or returns a `CustomError`. - /// Note that `node_data` can be modified to allow storing maintaining computation results on a per-object basis. + /// Note that `node_data` can be modified to allow storing maintaining computation results on a per-object basis. It should contain + /// its own mutable per-thread data as required. /// /// This method returns a vector of all tree items, along with their potentially modified custom node data. /// /// _Note_ that this method consumed the Tree to assure safe parallel traversal with mutation support. - pub fn traverse<F, P1, P2, MBFN, S, E>( + pub fn traverse<F, P1, P2, MBFN, E, R>( mut self, resolve: F, + resolve_data: &R, pack_entries_end: u64, - new_thread_state: impl Fn() -> S + Send + Clone, inspect_object: MBFN, Options { thread_limit, - object_progress, + mut object_progress, mut size_progress, should_interrupt, object_hash, }: Options<'_, P1, P2>, ) -> Result<Outcome<T>, Error> where - F: for<'r> Fn(EntryRange, &'r mut Vec<u8>) -> Option<()> + Send + Clone, + F: for<'r> Fn(EntryRange, &'r R) -> Option<&'r [u8]> + Send + Clone, + R: Send + Sync, P1: Progress, P2: Progress, - MBFN: Fn(&mut T, &mut <P1 as Progress>::SubProgress, Context<'_, S>) -> Result<(), E> + Send + Clone, + MBFN: FnMut(&mut T, &<P1 as Progress>::SubProgress, Context<'_>) -> Result<(), E> + Send + Clone, E: std::error::Error + Send + Sync + 'static, { self.set_pack_entries_end_and_resolve_ref_offsets(pack_entries_end)?; - let object_progress = OwnShared::new(Mutable::new(object_progress)); let num_objects = self.num_items(); let object_counter = { - let mut progress = lock(&object_progress); + let progress = &mut object_progress; progress.init(Some(num_objects), progress::count("objects")); progress.counter() }; size_progress.init(None, progress::bytes()); let size_counter = size_progress.counter(); let child_items = self.child_items.as_mut_slice(); + let object_progress = OwnShared::new(Mutable::new(object_progress)); let start = std::time::Instant::now(); in_parallel_with_slice( &mut self.root_items, thread_limit, { - let object_progress = object_progress.clone(); - let child_items = ItemSliceSend(child_items as *mut [Item<T>]); - move |thread_index| { - ( - Vec::<u8>::with_capacity(4096), - lock(&object_progress) - .add_child_with_id(format!("thread {thread_index}"), gix_features::progress::UNKNOWN), - new_thread_state(), - resolve.clone(), - inspect_object.clone(), - ItemSliceSend(child_items.0), - ) + let child_items = ItemSliceSend(std::ptr::slice_from_raw_parts_mut( + child_items.as_mut_ptr(), + child_items.len(), + )); + { + let object_progress = object_progress.clone(); + move |thread_index| { + let _ = &child_items; + resolve::State { + delta_bytes: Vec::<u8>::with_capacity(4096), + fully_resolved_delta_bytes: Vec::<u8>::with_capacity(4096), + progress: threading::lock(&object_progress).add_child(format!("thread {thread_index}")), + resolve: resolve.clone(), + modify_base: inspect_object.clone(), + child_items: child_items.clone(), + } + } } }, { - move |node, state| { + move |node, state, threads_left, should_interrupt| { resolve::deltas( object_counter.clone(), size_counter.clone(), node, state, + resolve_data, object_hash.len_in_bytes(), + threads_left, + should_interrupt, ) } }, @@ -166,7 +176,7 @@ where |_| (), )?; - lock(&object_progress).show_throughput(start); + threading::lock(&object_progress).show_throughput(start); size_progress.show_throughput(start); Ok(Outcome { diff --git a/vendor/gix-pack/src/cache/delta/traverse/resolve.rs b/vendor/gix-pack/src/cache/delta/traverse/resolve.rs index fc94d87ef..0a4d29191 100644 --- a/vendor/gix-pack/src/cache/delta/traverse/resolve.rs +++ b/vendor/gix-pack/src/cache/delta/traverse/resolve.rs @@ -1,6 +1,9 @@ -use std::{cell::RefCell, collections::BTreeMap, sync::atomic::Ordering}; +use std::{ + collections::BTreeMap, + sync::atomic::{AtomicBool, AtomicIsize, Ordering}, +}; -use gix_features::{progress::Progress, zlib}; +use gix_features::{progress::Progress, threading, zlib}; use crate::{ cache::delta::{ @@ -10,47 +13,57 @@ use crate::{ }, Item, }, + data, data::EntryRange, }; -pub(crate) fn deltas<T, F, P, MBFN, S, E>( +pub(crate) struct State<P, F, MBFN, T: Send> { + pub delta_bytes: Vec<u8>, + pub fully_resolved_delta_bytes: Vec<u8>, + pub progress: P, + pub resolve: F, + pub modify_base: MBFN, + pub child_items: ItemSliceSend<Item<T>>, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn deltas<T, F, MBFN, E, R, P>( object_counter: Option<gix_features::progress::StepShared>, size_counter: Option<gix_features::progress::StepShared>, - node: &mut crate::cache::delta::Item<T>, - (bytes_buf, ref mut progress, state, resolve, modify_base, child_items): &mut ( - Vec<u8>, - P, - S, - F, - MBFN, - ItemSliceSend<Item<T>>, - ), + node: &mut Item<T>, + State { + delta_bytes, + fully_resolved_delta_bytes, + progress, + resolve, + modify_base, + child_items, + }: &mut State<P, F, MBFN, T>, + resolve_data: &R, hash_len: usize, + threads_left: &AtomicIsize, + should_interrupt: &AtomicBool, ) -> Result<(), Error> where T: Send, - F: for<'r> Fn(EntryRange, &'r mut Vec<u8>) -> Option<()>, + R: Send + Sync, P: Progress, - MBFN: Fn(&mut T, &mut P, Context<'_, S>) -> Result<(), E>, + F: for<'r> Fn(EntryRange, &'r R) -> Option<&'r [u8]> + Send + Clone, + MBFN: FnMut(&mut T, &P, Context<'_>) -> Result<(), E> + Send + Clone, E: std::error::Error + Send + Sync + 'static, { let mut decompressed_bytes_by_pack_offset = BTreeMap::new(); - let bytes_buf = RefCell::new(bytes_buf); - let decompress_from_resolver = |slice: EntryRange| -> Result<(crate::data::Entry, u64, Vec<u8>), Error> { - let mut bytes_buf = bytes_buf.borrow_mut(); - bytes_buf.resize((slice.end - slice.start) as usize, 0); - resolve(slice.clone(), &mut bytes_buf).ok_or(Error::ResolveFailed { + let decompress_from_resolver = |slice: EntryRange, out: &mut Vec<u8>| -> Result<(data::Entry, u64), Error> { + let bytes = resolve(slice.clone(), resolve_data).ok_or(Error::ResolveFailed { pack_offset: slice.start, })?; - let entry = crate::data::Entry::from_bytes(&bytes_buf, slice.start, hash_len); - let compressed = &bytes_buf[entry.header_size()..]; + let entry = data::Entry::from_bytes(bytes, slice.start, hash_len); + let compressed = &bytes[entry.header_size()..]; let decompressed_len = entry.decompressed_size as usize; - Ok((entry, slice.end, decompress_all_at_once(compressed, decompressed_len)?)) + decompress_all_at_once_with(compressed, decompressed_len, out)?; + Ok((entry, slice.end)) }; - // Traverse the tree breadth first and loose the data produced for the base as it won't be needed anymore. - progress.init(None, gix_features::progress::count_with_decimals("objects", 2)); - // each node is a base, and its children always start out as deltas which become a base after applying them. // These will be pushed onto our stack until all are processed let root_level = 0; @@ -58,12 +71,17 @@ where root_level, Node { item: node, - child_items: child_items.0, + child_items: child_items.clone(), }, )]; while let Some((level, mut base)) = nodes.pop() { + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } let (base_entry, entry_end, base_bytes) = if level == root_level { - decompress_from_resolver(base.entry_slice())? + let mut buf = Vec::new(); + let (a, b) = decompress_from_resolver(base.entry_slice(), &mut buf)?; + (a, b, buf) } else { decompressed_bytes_by_pack_offset .remove(&base.offset()) @@ -81,7 +99,6 @@ where entry: &base_entry, entry_end, decompressed: &base_bytes, - state, level, }, ) @@ -93,20 +110,19 @@ where } for mut child in base.into_child_iter() { - let (mut child_entry, entry_end, delta_bytes) = decompress_from_resolver(child.entry_slice())?; - let (base_size, consumed) = crate::data::delta::decode_header_size(&delta_bytes); + let (mut child_entry, entry_end) = decompress_from_resolver(child.entry_slice(), delta_bytes)?; + let (base_size, consumed) = data::delta::decode_header_size(delta_bytes); let mut header_ofs = consumed; assert_eq!( base_bytes.len(), base_size as usize, - "recorded base size in delta does not match" + "recorded base size in delta does match the actual one" ); - let (result_size, consumed) = crate::data::delta::decode_header_size(&delta_bytes[consumed..]); + let (result_size, consumed) = data::delta::decode_header_size(&delta_bytes[consumed..]); header_ofs += consumed; - let mut fully_resolved_delta_bytes = bytes_buf.borrow_mut(); - fully_resolved_delta_bytes.resize(result_size as usize, 0); - crate::data::delta::apply(&base_bytes, &mut fully_resolved_delta_bytes, &delta_bytes[header_ofs..]); + set_len(fully_resolved_delta_bytes, result_size as usize); + data::delta::apply(&base_bytes, fully_resolved_delta_bytes, &delta_bytes[header_ofs..]); // FIXME: this actually invalidates the "pack_offset()" computation, which is not obvious to consumers // at all @@ -114,7 +130,7 @@ where if child.has_children() { decompressed_bytes_by_pack_offset.insert( child.offset(), - (child_entry, entry_end, fully_resolved_delta_bytes.to_owned()), + (child_entry, entry_end, std::mem::take(fully_resolved_delta_bytes)), ); nodes.push((level + 1, child)); } else { @@ -124,8 +140,7 @@ where Context { entry: &child_entry, entry_end, - decompressed: &fully_resolved_delta_bytes, - state, + decompressed: fully_resolved_delta_bytes, level: level + 1, }, ) @@ -136,19 +151,273 @@ where .map(|c| c.fetch_add(base_bytes.len(), Ordering::SeqCst)); } } + + // After the first round, see if we can use additional threads, and if so we enter multi-threaded mode. + // In it we will keep using new threads as they become available while using this thread for coordination. + // We optimize for a low memory footprint as we are likely to get here if long delta-chains with large objects are involved. + // Try to avoid going into threaded mode if there isn't more than one unit of work anyway. + if nodes.len() > 1 { + if let Ok(initial_threads) = + threads_left.fetch_update(Ordering::SeqCst, Ordering::SeqCst, |threads_available| { + (threads_available > 0).then_some(0) + }) + { + // Assure no memory is held here. + *delta_bytes = Vec::new(); + *fully_resolved_delta_bytes = Vec::new(); + return deltas_mt( + initial_threads, + decompressed_bytes_by_pack_offset, + object_counter, + size_counter, + progress, + nodes, + resolve.clone(), + resolve_data, + modify_base.clone(), + hash_len, + threads_left, + should_interrupt, + ); + } + } } Ok(()) } -fn decompress_all_at_once(b: &[u8], decompressed_len: usize) -> Result<Vec<u8>, Error> { - let mut out = Vec::new(); - out.resize(decompressed_len, 0); - zlib::Inflate::default() - .once(b, &mut out) - .map_err(|err| Error::ZlibInflate { +/// * `initial_threads` is the threads we may spawn, not accounting for our own thread which is still considered used by the parent +/// system. Since this thread will take a controlling function, we may spawn one more than that. In threaded mode, we will finish +/// all remaining work. +#[allow(clippy::too_many_arguments)] +pub(crate) fn deltas_mt<T, F, MBFN, E, R, P>( + mut threads_to_create: isize, + decompressed_bytes_by_pack_offset: BTreeMap<u64, (data::Entry, u64, Vec<u8>)>, + object_counter: Option<gix_features::progress::StepShared>, + size_counter: Option<gix_features::progress::StepShared>, + progress: &P, + nodes: Vec<(u16, Node<'_, T>)>, + resolve: F, + resolve_data: &R, + modify_base: MBFN, + hash_len: usize, + threads_left: &AtomicIsize, + should_interrupt: &AtomicBool, +) -> Result<(), Error> +where + T: Send, + R: Send + Sync, + P: Progress, + F: for<'r> Fn(EntryRange, &'r R) -> Option<&'r [u8]> + Send + Clone, + MBFN: FnMut(&mut T, &P, Context<'_>) -> Result<(), E> + Send + Clone, + E: std::error::Error + Send + Sync + 'static, +{ + let nodes = gix_features::threading::Mutable::new(nodes); + let decompressed_bytes_by_pack_offset = gix_features::threading::Mutable::new(decompressed_bytes_by_pack_offset); + threads_to_create += 1; // ourselves + let mut returned_ourselves = false; + + gix_features::parallel::threads(|s| -> Result<(), Error> { + let mut threads = Vec::new(); + let poll_interval = std::time::Duration::from_millis(100); + loop { + for tid in 0..threads_to_create { + let thread = gix_features::parallel::build_thread() + .name(format!("gix-pack.traverse_deltas.{tid}")) + .spawn_scoped(s, { + let nodes = &nodes; + let decompressed_bytes_by_pack_offset = &decompressed_bytes_by_pack_offset; + let resolve = resolve.clone(); + let mut modify_base = modify_base.clone(); + let object_counter = object_counter.as_ref(); + let size_counter = size_counter.as_ref(); + + move || -> Result<(), Error> { + let mut fully_resolved_delta_bytes = Vec::new(); + let mut delta_bytes = Vec::new(); + let decompress_from_resolver = + |slice: EntryRange, out: &mut Vec<u8>| -> Result<(data::Entry, u64), Error> { + let bytes = resolve(slice.clone(), resolve_data).ok_or(Error::ResolveFailed { + pack_offset: slice.start, + })?; + let entry = data::Entry::from_bytes(bytes, slice.start, hash_len); + let compressed = &bytes[entry.header_size()..]; + let decompressed_len = entry.decompressed_size as usize; + decompress_all_at_once_with(compressed, decompressed_len, out)?; + Ok((entry, slice.end)) + }; + + loop { + let (level, mut base) = match threading::lock(nodes).pop() { + Some(v) => v, + None => break, + }; + if should_interrupt.load(Ordering::Relaxed) { + return Err(Error::Interrupted); + } + let (base_entry, entry_end, base_bytes) = if level == 0 { + let mut buf = Vec::new(); + let (a, b) = decompress_from_resolver(base.entry_slice(), &mut buf)?; + (a, b, buf) + } else { + threading::lock(decompressed_bytes_by_pack_offset) + .remove(&base.offset()) + .expect("we store the resolved delta buffer when done") + }; + + // anything done here must be repeated further down for leaf-nodes. + // This way we avoid retaining their decompressed memory longer than needed (they have no children, + // thus their memory can be released right away, using 18% less peak memory on the linux kernel). + { + modify_base( + base.data(), + progress, + Context { + entry: &base_entry, + entry_end, + decompressed: &base_bytes, + level, + }, + ) + .map_err(|err| Box::new(err) as Box<dyn std::error::Error + Send + Sync>)?; + object_counter.as_ref().map(|c| c.fetch_add(1, Ordering::SeqCst)); + size_counter + .as_ref() + .map(|c| c.fetch_add(base_bytes.len(), Ordering::SeqCst)); + } + + for mut child in base.into_child_iter() { + let (mut child_entry, entry_end) = + decompress_from_resolver(child.entry_slice(), &mut delta_bytes)?; + let (base_size, consumed) = data::delta::decode_header_size(&delta_bytes); + let mut header_ofs = consumed; + assert_eq!( + base_bytes.len(), + base_size as usize, + "recorded base size in delta does match the actual one" + ); + let (result_size, consumed) = + data::delta::decode_header_size(&delta_bytes[consumed..]); + header_ofs += consumed; + + fully_resolved_delta_bytes.resize(result_size as usize, 0); + data::delta::apply( + &base_bytes, + &mut fully_resolved_delta_bytes, + &delta_bytes[header_ofs..], + ); + + // FIXME: this actually invalidates the "pack_offset()" computation, which is not obvious to consumers + // at all + child_entry.header = base_entry.header; // assign the actual object type, instead of 'delta' + if child.has_children() { + threading::lock(decompressed_bytes_by_pack_offset).insert( + child.offset(), + (child_entry, entry_end, std::mem::take(&mut fully_resolved_delta_bytes)), + ); + threading::lock(nodes).push((level + 1, child)); + } else { + modify_base( + child.data(), + progress, + Context { + entry: &child_entry, + entry_end, + decompressed: &fully_resolved_delta_bytes, + level: level + 1, + }, + ) + .map_err(|err| Box::new(err) as Box<dyn std::error::Error + Send + Sync>)?; + object_counter.as_ref().map(|c| c.fetch_add(1, Ordering::SeqCst)); + size_counter + .as_ref() + .map(|c| c.fetch_add(base_bytes.len(), Ordering::SeqCst)); + } + } + } + Ok(()) + } + })?; + threads.push(thread); + } + if threads_left + .fetch_update(Ordering::SeqCst, Ordering::SeqCst, |threads_available: isize| { + (threads_available > 0).then(|| { + threads_to_create = threads_available.min(threading::lock(&nodes).len() as isize); + threads_available - threads_to_create + }) + }) + .is_err() + { + threads_to_create = 0; + } + + // What we really want to do is either wait for one of our threads to go down + // or for another scheduled thread to become available. Unfortunately we can't do that, + // but may instead find a good way to set the polling interval instead of hard-coding it. + std::thread::sleep(poll_interval); + // Get out of threads are already starving or they would be starving soon as no work is left. + if threads.iter().any(|t| t.is_finished()) { + let mut running_threads = Vec::new(); + for thread in threads.drain(..) { + if thread.is_finished() { + match thread.join() { + Ok(Err(err)) => return Err(err), + Ok(Ok(())) => { + if !returned_ourselves { + returned_ourselves = true; + } else { + threads_left.fetch_add(1, Ordering::SeqCst); + } + } + Err(err) => { + std::panic::resume_unwind(err); + } + } + } else { + running_threads.push(thread); + } + } + if running_threads.is_empty() && threading::lock(&nodes).is_empty() { + break; + } + threads = running_threads; + } + } + + Ok(()) + }) +} + +fn set_len(v: &mut Vec<u8>, new_len: usize) { + if new_len > v.len() { + v.reserve_exact(new_len.saturating_sub(v.capacity()) + (v.capacity() - v.len())); + // SAFETY: + // 1. we have reserved enough capacity to fit `new_len` + // 2. the caller is trusted to write into `v` to completely fill `new_len`. + #[allow(unsafe_code, clippy::uninit_vec)] + unsafe { + v.set_len(new_len); + } + } else { + v.truncate(new_len) + } +} + +fn decompress_all_at_once_with(b: &[u8], decompressed_len: usize, out: &mut Vec<u8>) -> Result<(), Error> { + set_len(out, decompressed_len); + use std::cell::RefCell; + thread_local! { + pub static INFLATE: RefCell<zlib::Inflate> = RefCell::new(zlib::Inflate::default()); + } + + INFLATE.with(|inflate| { + let mut inflate = inflate.borrow_mut(); + inflate.reset(); + inflate.once(b, out).map_err(|err| Error::ZlibInflate { source: err, message: "Failed to decompress entry", - })?; - Ok(out) + }) + })?; + Ok(()) } diff --git a/vendor/gix-pack/src/cache/delta/traverse/util.rs b/vendor/gix-pack/src/cache/delta/traverse/util.rs index e7caf2ff5..1b7015351 100644 --- a/vendor/gix-pack/src/cache/delta/traverse/util.rs +++ b/vendor/gix-pack/src/cache/delta/traverse/util.rs @@ -4,6 +4,8 @@ pub struct ItemSliceSend<T>(pub *mut [T]) where T: Send; +/// SAFETY: This would be unsafe if this would ever be abused, but it's used internally and only in a way that assure that the pointers +/// don't violate aliasing rules. impl<T> Clone for ItemSliceSend<T> where T: Send, @@ -18,12 +20,12 @@ where unsafe impl<T> Send for ItemSliceSend<T> where T: Send {} /// An item returned by `iter_root_chunks`, allowing access to the `data` stored alongside nodes in a [`Tree`]. -pub struct Node<'a, T> { +pub struct Node<'a, T: Send> { pub item: &'a mut Item<T>, - pub child_items: *mut [Item<T>], + pub child_items: ItemSliceSend<Item<T>>, } -impl<'a, T> Node<'a, T> { +impl<'a, T: Send> Node<'a, T> { /// Returns the offset into the pack at which the `Node`s data is located. pub fn offset(&self) -> u64 { self.item.offset @@ -55,8 +57,8 @@ impl<'a, T> Node<'a, T> { // SAFETY: The resulting mutable pointer cannot be yielded by any other node. #[allow(unsafe_code)] Node { - item: unsafe { &mut *(children as *mut Item<T>).add(index as usize) }, - child_items: children, + item: &mut unsafe { &mut *children.0 }[index as usize], + child_items: children.clone(), } }) } diff --git a/vendor/gix-pack/src/cache/lru.rs b/vendor/gix-pack/src/cache/lru.rs index bba4f5d33..5b156141e 100644 --- a/vendor/gix-pack/src/cache/lru.rs +++ b/vendor/gix-pack/src/cache/lru.rs @@ -48,24 +48,26 @@ mod memory { impl DecodeEntry for MemoryCappedHashmap { fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize) { self.debug.put(); - if let Ok(Some(previous_entry)) = self.inner.put_with_weight( + let res = self.inner.put_with_weight( (pack_id, offset), Entry { - data: self - .free_list - .pop() - .map(|mut v| { + data: self.free_list.pop().map_or_else( + || Vec::from(data), + |mut v| { v.clear(); v.resize(data.len(), 0); v.copy_from_slice(data); v - }) - .unwrap_or_else(|| Vec::from(data)), + }, + ), kind, compressed_size, }, - ) { - self.free_list.push(previous_entry.data) + ); + match res { + Ok(Some(previous_entry)) => self.free_list.push(previous_entry.data), + Ok(None) => {} + Err((_key, value)) => self.free_list.push(value.data), } } @@ -104,41 +106,74 @@ mod _static { /// Values of 64 seem to improve performance. pub struct StaticLinkedList<const SIZE: usize> { inner: uluru::LRUCache<Entry, SIZE>, - free_list: Vec<Vec<u8>>, + last_evicted: Vec<u8>, debug: gix_features::cache::Debug, + /// the amount of bytes we are currently holding, taking into account the capacities of all Vecs we keep. + mem_used: usize, + /// The total amount of memory we should be able to hold with all entries combined. + mem_limit: usize, } - impl<const SIZE: usize> Default for StaticLinkedList<SIZE> { - fn default() -> Self { + impl<const SIZE: usize> StaticLinkedList<SIZE> { + /// Create a new list with a memory limit of `mem_limit` in bytes. If 0, there is no memory limit. + pub fn new(mem_limit: usize) -> Self { StaticLinkedList { inner: Default::default(), - free_list: Vec::new(), + last_evicted: Vec::new(), debug: gix_features::cache::Debug::new(format!("StaticLinkedList<{SIZE}>")), + mem_used: 0, + mem_limit: if mem_limit == 0 { usize::MAX } else { mem_limit }, } } } + impl<const SIZE: usize> Default for StaticLinkedList<SIZE> { + fn default() -> Self { + Self::new(96 * 1024 * 1024) + } + } + impl<const SIZE: usize> DecodeEntry for StaticLinkedList<SIZE> { fn put(&mut self, pack_id: u32, offset: u64, data: &[u8], kind: gix_object::Kind, compressed_size: usize) { + // We cannot possibly hold this much. + if data.len() > self.mem_limit { + return; + } + // If we could hold it but are are at limit, all we can do is make space. + let mem_free = self.mem_limit - self.mem_used; + if data.len() > mem_free { + // prefer freeing free-lists instead of clearing our cache + let free_list_cap = self.last_evicted.len(); + self.last_evicted = Vec::new(); + // still not enough? clear everything + if data.len() > mem_free + free_list_cap { + self.inner.clear(); + self.mem_used = 0; + } else { + self.mem_used -= free_list_cap; + } + } self.debug.put(); + let (prev_cap, cur_cap); if let Some(previous) = self.inner.insert(Entry { offset, pack_id, - data: self - .free_list - .pop() - .map(|mut v| { - v.clear(); - v.resize(data.len(), 0); - v.copy_from_slice(data); - v - }) - .unwrap_or_else(|| Vec::from(data)), + data: { + let mut v = std::mem::take(&mut self.last_evicted); + prev_cap = v.capacity(); + v.clear(); + v.resize(data.len(), 0); + v.copy_from_slice(data); + cur_cap = v.capacity(); + v + }, kind, compressed_size, }) { - self.free_list.push(previous.data) + // No need to adjust capacity as we already counted it. + self.last_evicted = previous.data; } + self.mem_used = self.mem_used + cur_cap - prev_cap; } fn get(&mut self, pack_id: u32, offset: u64, out: &mut Vec<u8>) -> Option<(gix_object::Kind, usize)> { @@ -159,6 +194,56 @@ mod _static { res } } + + #[cfg(test)] + mod tests { + use super::*; + + #[test] + fn no_limit() { + let c = StaticLinkedList::<10>::new(0); + assert_eq!( + c.mem_limit, + usize::MAX, + "zero is automatically turned into a large limit that is equivalent to unlimited" + ); + } + + #[test] + fn journey() { + let mut c = StaticLinkedList::<10>::new(100); + assert_eq!(c.mem_limit, 100); + assert_eq!(c.mem_used, 0); + + // enough memory for normal operation + let mut last_mem_used = 0; + for _ in 0..10 { + c.put(0, 0, &[0], gix_object::Kind::Blob, 1); + assert!(c.mem_used > last_mem_used); + last_mem_used = c.mem_used; + } + assert_eq!(c.mem_used, 80, "there is a minimal vec size"); + assert_eq!(c.inner.len(), 10); + assert_eq!(c.last_evicted.len(), 0); + + c.put(0, 0, &(0..20).collect::<Vec<_>>(), gix_object::Kind::Blob, 1); + assert_eq!(c.inner.len(), 10); + assert_eq!(c.mem_used, 80 + 20); + assert_eq!(c.last_evicted.len(), 1); + + c.put(0, 0, &(0..50).collect::<Vec<_>>(), gix_object::Kind::Blob, 1); + assert_eq!(c.inner.len(), 1, "cache clearance wasn't necessary"); + assert_eq!(c.last_evicted.len(), 0, "the free list was cleared"); + assert_eq!(c.mem_used, 50); + + c.put(0, 0, &(0..101).collect::<Vec<_>>(), gix_object::Kind::Blob, 1); + assert_eq!( + c.inner.len(), + 1, + "objects that won't ever fit within the memory limit are ignored" + ); + } + } } #[cfg(feature = "pack-cache-lru-static")] diff --git a/vendor/gix-pack/src/cache/object.rs b/vendor/gix-pack/src/cache/object.rs index e64f47a8c..26896bf89 100644 --- a/vendor/gix-pack/src/cache/object.rs +++ b/vendor/gix-pack/src/cache/object.rs @@ -1,6 +1,4 @@ -//! # Note -//! -//! This module is a bit 'misplaced' if spelled out like 'gix_pack::cache::object::*' but is best placed here for code re-use and +//! This module is a bit 'misplaced' if spelled out like '`gix_pack::cache::object::`*' but is best placed here for code re-use and //! general usefulness. use crate::cache; @@ -58,23 +56,25 @@ mod memory { /// Put the object going by `id` of `kind` with `data` into the cache. fn put(&mut self, id: gix_hash::ObjectId, kind: gix_object::Kind, data: &[u8]) { self.debug.put(); - if let Ok(Some(previous_entry)) = self.inner.put_with_weight( + let res = self.inner.put_with_weight( id, Entry { - data: self - .free_list - .pop() - .map(|mut v| { + data: self.free_list.pop().map_or_else( + || Vec::from(data), + |mut v| { v.clear(); v.resize(data.len(), 0); v.copy_from_slice(data); v - }) - .unwrap_or_else(|| Vec::from(data)), + }, + ), kind, }, - ) { - self.free_list.push(previous_entry.data) + ); + match res { + Ok(Some(previous_entry)) => self.free_list.push(previous_entry.data), + Ok(None) => {} + Err((_key, value)) => self.free_list.push(value.data), } } diff --git a/vendor/gix-pack/src/data/file/decode/entry.rs b/vendor/gix-pack/src/data/file/decode/entry.rs index a467ccd8e..f82e33a7b 100644 --- a/vendor/gix-pack/src/data/file/decode/entry.rs +++ b/vendor/gix-pack/src/data/file/decode/entry.rs @@ -126,9 +126,18 @@ impl File { let offset: usize = data_offset.try_into().expect("offset representable by machine"); assert!(offset < self.data.len(), "entry offset out of bounds"); - zlib::Inflate::default() - .once(&self.data[offset..], out) - .map(|(_status, consumed_in, _consumed_out)| consumed_in) + use std::cell::RefCell; + thread_local! { + pub static INFLATE: RefCell<zlib::Inflate> = RefCell::new(zlib::Inflate::default()); + } + INFLATE.with(|inflate| { + let mut inflate = inflate.borrow_mut(); + let res = inflate + .once(&self.data[offset..], out) + .map(|(_status, consumed_in, _consumed_out)| consumed_in); + inflate.reset(); + res + }) } /// Like `decompress_entry_from_data_offset`, but returns consumed input and output. @@ -140,9 +149,19 @@ impl File { let offset: usize = data_offset.try_into().expect("offset representable by machine"); assert!(offset < self.data.len(), "entry offset out of bounds"); - zlib::Inflate::default() - .once(&self.data[offset..], out) - .map(|(_status, consumed_in, consumed_out)| (consumed_in, consumed_out)) + use std::cell::RefCell; + thread_local! { + pub static INFLATE: RefCell<zlib::Inflate> = RefCell::new(zlib::Inflate::default()); + } + + INFLATE.with(|inflate| { + let mut inflate = inflate.borrow_mut(); + let res = inflate + .once(&self.data[offset..], out) + .map(|(_status, consumed_in, consumed_out)| (consumed_in, consumed_out)); + inflate.reset(); + res + }) } /// Decode an entry, resolving delta's as needed, while growing the `out` vector if there is not enough @@ -215,6 +234,8 @@ impl File { } break; } + // This is a pessimistic guess, as worst possible compression should not be bigger than the data itself. + // TODO: is this assumption actually true? total_delta_data_size += cursor.decompressed_size; let decompressed_size = cursor .decompressed_size diff --git a/vendor/gix-pack/src/data/input/bytes_to_entries.rs b/vendor/gix-pack/src/data/input/bytes_to_entries.rs index cf20d5fbf..995c8df2c 100644 --- a/vendor/gix-pack/src/data/input/bytes_to_entries.rs +++ b/vendor/gix-pack/src/data/input/bytes_to_entries.rs @@ -11,7 +11,7 @@ use crate::data::input; /// An iterator over [`Entries`][input::Entry] in a byte stream. /// -/// The iterator used as part of [Bundle::write_to_directory(…)][crate::Bundle::write_to_directory()]. +/// The iterator used as part of [`Bundle::write_to_directory(…)`][crate::Bundle::write_to_directory()]. pub struct BytesToEntriesIter<BR> { read: BR, decompressor: Option<Box<Decompress>>, diff --git a/vendor/gix-pack/src/data/input/lookup_ref_delta_objects.rs b/vendor/gix-pack/src/data/input/lookup_ref_delta_objects.rs index f52c645f8..d95e6176d 100644 --- a/vendor/gix-pack/src/data/input/lookup_ref_delta_objects.rs +++ b/vendor/gix-pack/src/data/input/lookup_ref_delta_objects.rs @@ -193,7 +193,7 @@ where fn size_hint(&self) -> (usize, Option<usize>) { let (min, max) = self.inner.size_hint(); - max.map(|max| (min, Some(max * 2))).unwrap_or_else(|| (min * 2, None)) + max.map_or_else(|| (min * 2, None), |max| (min, Some(max * 2))) } } diff --git a/vendor/gix-pack/src/data/output/bytes.rs b/vendor/gix-pack/src/data/output/bytes.rs index ec219db9d..6616bd14a 100644 --- a/vendor/gix-pack/src/data/output/bytes.rs +++ b/vendor/gix-pack/src/data/output/bytes.rs @@ -51,7 +51,7 @@ where /// `output` writer, resembling a pack of `version` with exactly `num_entries` amount of objects contained in it. /// `object_hash` is the kind of hash to use for the pack checksum and maybe other places, depending on the version. /// - /// The input chunks are expected to be sorted already. You can use the [InOrderIter][gix_features::parallel::InOrderIter] to assure + /// The input chunks are expected to be sorted already. You can use the [`InOrderIter`][gix_features::parallel::InOrderIter] to assure /// this happens on the fly holding entire chunks in memory as long as needed for them to be dispensed in order. /// /// # Panics diff --git a/vendor/gix-pack/src/data/output/count/mod.rs b/vendor/gix-pack/src/data/output/count/mod.rs index bec75d6f3..0c33abd97 100644 --- a/vendor/gix-pack/src/data/output/count/mod.rs +++ b/vendor/gix-pack/src/data/output/count/mod.rs @@ -13,14 +13,14 @@ pub enum PackLocation { } impl PackLocation { - /// Directly go through to LookedUp variant, panic otherwise + /// Directly go through to `LookedUp` variant, panic otherwise pub fn is_none(&self) -> bool { match self { PackLocation::LookedUp(opt) => opt.is_none(), PackLocation::NotLookedUp => unreachable!("must have been resolved"), } } - /// Directly go through to LookedUp variant, panic otherwise + /// Directly go through to `LookedUp` variant, panic otherwise pub fn as_ref(&self) -> Option<&crate::data::entry::Location> { match self { PackLocation::LookedUp(opt) => opt.as_ref(), diff --git a/vendor/gix-pack/src/data/output/count/objects/mod.rs b/vendor/gix-pack/src/data/output/count/objects/mod.rs index d56bc9a5f..a13e41146 100644 --- a/vendor/gix-pack/src/data/output/count/objects/mod.rs +++ b/vendor/gix-pack/src/data/output/count/objects/mod.rs @@ -329,7 +329,7 @@ mod expand { &mut traverse_delegate, ) .map_err(Error::TreeTraverse)?; - for id in traverse_delegate.non_trees.iter() { + for id in &traverse_delegate.non_trees { out.push(id_to_count(db, buf1, id, progress, stats, allow_pack_lookups)); } break; diff --git a/vendor/gix-pack/src/data/output/count/objects/types.rs b/vendor/gix-pack/src/data/output/count/objects/types.rs index cd9e92c7a..f39a24ee4 100644 --- a/vendor/gix-pack/src/data/output/count/objects/types.rs +++ b/vendor/gix-pack/src/data/output/count/objects/types.rs @@ -77,7 +77,7 @@ impl Default for Options { } } -/// The error returned by the pack generation iterator [bytes::FromEntriesIter][crate::data::output::bytes::FromEntriesIter]. +/// The error returned by the pack generation iterator [`bytes::FromEntriesIter`][crate::data::output::bytes::FromEntriesIter]. #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error<FindErr, IterErr> diff --git a/vendor/gix-pack/src/index/access.rs b/vendor/gix-pack/src/index/access.rs index c384109ac..18fb70e2a 100644 --- a/vendor/gix-pack/src/index/access.rs +++ b/vendor/gix-pack/src/index/access.rs @@ -64,7 +64,7 @@ impl index::File { } /// Returns the object hash at the given index in our list of (sorted) sha1 hashes. - /// The index ranges from 0 to self.num_objects() + /// The index ranges from 0 to `self.num_objects()` /// /// # Panics /// diff --git a/vendor/gix-pack/src/index/traverse/mod.rs b/vendor/gix-pack/src/index/traverse/mod.rs index 68a502bca..83173f904 100644 --- a/vendor/gix-pack/src/index/traverse/mod.rs +++ b/vendor/gix-pack/src/index/traverse/mod.rs @@ -1,6 +1,9 @@ use std::sync::atomic::AtomicBool; -use gix_features::{parallel, progress::Progress}; +use gix_features::{ + parallel, + progress::{Progress, RawProgress}, +}; use crate::index; @@ -79,7 +82,7 @@ impl index::File { pack: &crate::data::File, progress: P, should_interrupt: &AtomicBool, - new_processor: impl Fn() -> Processor + Send + Clone, + processor: Processor, Options { traversal, thread_limit, @@ -91,17 +94,12 @@ impl index::File { P: Progress, C: crate::cache::DecodeEntry, E: std::error::Error + Send + Sync + 'static, - Processor: FnMut( - gix_object::Kind, - &[u8], - &index::Entry, - &mut <P::SubProgress as Progress>::SubProgress, - ) -> Result<(), E>, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn RawProgress) -> Result<(), E> + Send + Clone, F: Fn() -> C + Send + Clone, { match traversal { Algorithm::Lookup => self.traverse_with_lookup( - new_processor, + processor, pack, progress, should_interrupt, @@ -113,10 +111,10 @@ impl index::File { ), Algorithm::DeltaTreeLookup => self.traverse_with_index( pack, - new_processor, + processor, progress, should_interrupt, - crate::index::traverse::with_index::Options { check, thread_limit }, + with_index::Options { check, thread_limit }, ), } } @@ -151,19 +149,18 @@ impl index::File { } #[allow(clippy::too_many_arguments)] - fn decode_and_process_entry<C, P, E>( + fn decode_and_process_entry<C, E>( &self, check: SafetyCheck, pack: &crate::data::File, cache: &mut C, buf: &mut Vec<u8>, - progress: &mut P, - index_entry: &crate::index::Entry, - processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>, + progress: &mut dyn RawProgress, + index_entry: &index::Entry, + processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn RawProgress) -> Result<(), E>, ) -> Result<crate::data::decode::entry::Outcome, Error<E>> where C: crate::cache::DecodeEntry, - P: Progress, E: std::error::Error + Send + Sync + 'static, { let pack_entry = pack.entry(index_entry.pack_offset); @@ -192,9 +189,9 @@ impl index::File { check, object_kind, buf, - progress, index_entry, || pack.entry_crc32(index_entry.pack_offset, entry_len), + progress, processor, )?; Ok(entry_stats) @@ -202,17 +199,16 @@ impl index::File { } #[allow(clippy::too_many_arguments)] -fn process_entry<P, E>( +fn process_entry<E>( check: SafetyCheck, object_kind: gix_object::Kind, decompressed: &[u8], - progress: &mut P, - index_entry: &crate::index::Entry, + index_entry: &index::Entry, pack_entry_crc32: impl FnOnce() -> u32, - processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &mut P) -> Result<(), E>, + progress: &dyn RawProgress, + processor: &mut impl FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn RawProgress) -> Result<(), E>, ) -> Result<(), Error<E>> where - P: Progress, E: std::error::Error + Send + Sync + 'static, { if check.object_checksum() { diff --git a/vendor/gix-pack/src/index/traverse/with_index.rs b/vendor/gix-pack/src/index/traverse/with_index.rs index 769bbd07f..884277c9d 100644 --- a/vendor/gix-pack/src/index/traverse/with_index.rs +++ b/vendor/gix-pack/src/index/traverse/with_index.rs @@ -59,19 +59,16 @@ impl index::File { pub fn traverse_with_index<P, Processor, E>( &self, pack: &crate::data::File, - new_processor: impl Fn() -> Processor + Send + Clone, + mut processor: Processor, mut progress: P, should_interrupt: &AtomicBool, Options { check, thread_limit }: Options, ) -> Result<Outcome<P>, Error<E>> where P: Progress, - Processor: FnMut( - gix_object::Kind, - &[u8], - &index::Entry, - &mut <P::SubProgress as Progress>::SubProgress, - ) -> Result<(), E>, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn gix_features::progress::RawProgress) -> Result<(), E> + + Send + + Clone, E: std::error::Error + Send + Sync + 'static, { let (verify_result, traversal_result) = parallel::join( @@ -113,29 +110,27 @@ impl index::File { self.object_hash, )?; let mut outcome = digest_statistics(tree.traverse( - |slice, out| pack.entry_slice(slice).map(|entry| out.copy_from_slice(entry)), + |slice, pack| pack.entry_slice(slice), + pack, pack.pack_end() as u64, - new_processor, - |data, - progress, - traverse::Context { - entry: pack_entry, - entry_end, - decompressed: bytes, - state: ref mut processor, - level, - }| { + move |data, + progress, + traverse::Context { + entry: pack_entry, + entry_end, + decompressed: bytes, + level, + }| { let object_kind = pack_entry.header.as_kind().expect("non-delta object"); data.level = level; data.decompressed_size = pack_entry.decompressed_size; data.object_kind = object_kind; data.compressed_size = entry_end - pack_entry.data_offset; data.object_size = bytes.len() as u64; - let result = crate::index::traverse::process_entry( + let result = index::traverse::process_entry( check, object_kind, bytes, - progress, &data.index_entry, || { // TODO: Fix this - we overwrite the header of 'data' which also changes the computed entry size, @@ -146,7 +141,8 @@ impl index::File { .expect("slice pointing into the pack (by now data is verified)"), ) }, - processor, + progress, + &mut processor, ); match result { Err(err @ Error::PackDecode { .. }) if !check.fatal_decode_error() => { @@ -156,7 +152,7 @@ impl index::File { res => res, } }, - crate::cache::delta::traverse::Options { + traverse::Options { object_progress: progress.add_child_with_id("Resolving", ProgressId::DecodedObjects.into()), size_progress: progress.add_child_with_id("Decoding", ProgressId::DecodedBytes.into()), thread_limit, diff --git a/vendor/gix-pack/src/index/traverse/with_lookup.rs b/vendor/gix-pack/src/index/traverse/with_lookup.rs index 509ae4e4f..0165e4e01 100644 --- a/vendor/gix-pack/src/index/traverse/with_lookup.rs +++ b/vendor/gix-pack/src/index/traverse/with_lookup.rs @@ -67,8 +67,8 @@ impl index::File { /// For more details, see the documentation on the [`traverse()`][index::File::traverse()] method. pub fn traverse_with_lookup<P, C, Processor, E, F>( &self, - new_processor: impl Fn() -> Processor + Send + Clone, - pack: &crate::data::File, + mut processor: Processor, + pack: &data::File, mut progress: P, should_interrupt: &AtomicBool, Options { @@ -81,12 +81,9 @@ impl index::File { P: Progress, C: crate::cache::DecodeEntry, E: std::error::Error + Send + Sync + 'static, - Processor: FnMut( - gix_object::Kind, - &[u8], - &index::Entry, - &mut <P::SubProgress as Progress>::SubProgress, - ) -> Result<(), E>, + Processor: FnMut(gix_object::Kind, &[u8], &index::Entry, &dyn gix_features::progress::RawProgress) -> Result<(), E> + + Send + + Clone, F: Fn() -> C + Send + Clone, { let (verify_result, traversal_result) = parallel::join( @@ -133,7 +130,6 @@ impl index::File { move |index| { ( make_pack_lookup_cache(), - new_processor(), Vec::with_capacity(2048), // decode buffer lock(&reduce_progress) .add_child_with_id(format!("thread {index}"), gix_features::progress::UNKNOWN), // per thread progress @@ -146,9 +142,9 @@ impl index::File { input_chunks, thread_limit, state_per_thread, - |entries: &[index::Entry], - (cache, ref mut processor, buf, progress)| - -> Result<Vec<data::decode::entry::Outcome>, Error<_>> { + move |entries: &[index::Entry], + (cache, buf, progress)| + -> Result<Vec<data::decode::entry::Outcome>, Error<_>> { progress.init( Some(entries.len()), gix_features::progress::count_with_decimals("objects", 2), @@ -163,7 +159,7 @@ impl index::File { buf, progress, index_entry, - processor, + &mut processor, ); progress.inc(); let stat = match result { @@ -174,6 +170,9 @@ impl index::File { res => res, }?; stats.push(stat); + if should_interrupt.load(Ordering::Relaxed) { + break; + } } Ok(stats) }, diff --git a/vendor/gix-pack/src/index/verify.rs b/vendor/gix-pack/src/index/verify.rs index 182f816ba..6af352ac9 100644 --- a/vendor/gix-pack/src/index/verify.rs +++ b/vendor/gix-pack/src/index/verify.rs @@ -198,7 +198,7 @@ impl index::File { pack, progress, should_interrupt, - || { + { let mut encode_buf = Vec::with_capacity(2048); move |kind, data, index_entry, progress| { Self::verify_entry(verify_mode, &mut encode_buf, kind, data, index_entry, progress) @@ -231,17 +231,14 @@ impl index::File { } #[allow(clippy::too_many_arguments)] - fn verify_entry<P>( + fn verify_entry( verify_mode: Mode, encode_buf: &mut Vec<u8>, object_kind: gix_object::Kind, buf: &[u8], index_entry: &index::Entry, - progress: &mut P, - ) -> Result<(), integrity::Error> - where - P: Progress, - { + progress: &dyn gix_features::progress::RawProgress, + ) -> Result<(), integrity::Error> { if let Mode::HashCrc32Decode | Mode::HashCrc32DecodeEncode = verify_mode { use gix_object::Kind::*; match object_kind { @@ -260,7 +257,7 @@ impl index::File { .expect("writing to a memory buffer never fails"); if encode_buf.as_slice() != buf { let mut should_return_error = true; - if let gix_object::Kind::Tree = object_kind { + if let Tree = object_kind { if buf.as_bstr().find(b"100664").is_some() || buf.as_bstr().find(b"100640").is_some() { progress.info(format!("Tree object {} would be cleaned up during re-serialization, replacing mode '100664|100640' with '100644'", index_entry.oid)); should_return_error = false diff --git a/vendor/gix-pack/src/index/write/encode.rs b/vendor/gix-pack/src/index/write/encode.rs index 80f0cac61..f1195875c 100644 --- a/vendor/gix-pack/src/index/write/encode.rs +++ b/vendor/gix-pack/src/index/write/encode.rs @@ -111,10 +111,7 @@ pub(crate) fn fanout(iter: impl ExactSizeIterator<Item = u8>) -> [u32; 256] { entries_len } else { idx_and_entry = iter.find(|(_, first_byte)| *first_byte != byte); - upper_bound = idx_and_entry - .as_ref() - .map(|(idx, _)| *idx as u32) - .unwrap_or(entries_len); + upper_bound = idx_and_entry.as_ref().map_or(entries_len, |(idx, _)| *idx as u32); upper_bound } } diff --git a/vendor/gix-pack/src/index/write/mod.rs b/vendor/gix-pack/src/index/write/mod.rs index 39ed0f31e..72a076a85 100644 --- a/vendor/gix-pack/src/index/write/mod.rs +++ b/vendor/gix-pack/src/index/write/mod.rs @@ -83,20 +83,22 @@ impl crate::index::File { /// It should return `None` if the entry cannot be resolved from the pack that produced the `entries` iterator, causing /// the write operation to fail. #[allow(clippy::too_many_arguments)] - pub fn write_data_iter_to_stream<F, F2>( + pub fn write_data_iter_to_stream<F, F2, R, P>( version: crate::index::Version, make_resolver: F, entries: impl Iterator<Item = Result<crate::data::input::Entry, crate::data::input::Error>>, thread_limit: Option<usize>, - mut root_progress: impl Progress, + mut root_progress: P, out: impl io::Write, should_interrupt: &AtomicBool, object_hash: gix_hash::Kind, pack_version: crate::data::Version, ) -> Result<Outcome, Error> where - F: FnOnce() -> io::Result<F2>, - F2: for<'r> Fn(crate::data::EntryRange, &'r mut Vec<u8>) -> Option<()> + Send + Clone, + F: FnOnce() -> io::Result<(F2, R)>, + R: Send + Sync, + F2: for<'r> Fn(crate::data::EntryRange, &'r R) -> Option<&'r [u8]> + Send + Clone, + P: Progress, { if version != crate::index::Version::default() { return Err(Error::Unsupported(version)); @@ -180,12 +182,12 @@ impl crate::index::File { root_progress.inc(); - let resolver = make_resolver()?; + let (resolver, pack) = make_resolver()?; let sorted_pack_offsets_by_oid = { let traverse::Outcome { roots, children } = tree.traverse( resolver, + &pack, pack_entries_end, - || (), |data, _progress, traverse::Context { @@ -250,14 +252,7 @@ impl crate::index::File { } fn modify_base(entry: &mut TreeEntry, pack_entry: &crate::data::Entry, decompressed: &[u8], hash: gix_hash::Kind) { - fn compute_hash(kind: gix_object::Kind, bytes: &[u8], object_hash: gix_hash::Kind) -> gix_hash::ObjectId { - let mut hasher = gix_features::hash::hasher(object_hash); - hasher.update(&gix_object::encode::loose_header(kind, bytes.len())); - hasher.update(bytes); - gix_hash::ObjectId::from(hasher.digest()) - } - let object_kind = pack_entry.header.as_kind().expect("base object as source of iteration"); - let id = compute_hash(object_kind, decompressed, hash); + let id = gix_object::compute_hash(hash, object_kind, decompressed); entry.id = id; } diff --git a/vendor/gix-pack/src/multi_index/chunk.rs b/vendor/gix-pack/src/multi_index/chunk.rs index 7ed8eebcb..48a003ca0 100644 --- a/vendor/gix-pack/src/multi_index/chunk.rs +++ b/vendor/gix-pack/src/multi_index/chunk.rs @@ -11,7 +11,7 @@ pub mod index_names { pub mod decode { use gix_object::bstr::BString; - /// The error returned by [from_bytes()][super::from_bytes()]. + /// The error returned by [`from_bytes()`][super::from_bytes()]. #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { diff --git a/vendor/gix-pack/src/multi_index/write.rs b/vendor/gix-pack/src/multi_index/write.rs index 314506401..9002af9eb 100644 --- a/vendor/gix-pack/src/multi_index/write.rs +++ b/vendor/gix-pack/src/multi_index/write.rs @@ -10,7 +10,7 @@ use gix_features::progress::Progress; use crate::multi_index; mod error { - /// The error returned by [multi_index::File::write_from_index_paths()][super::multi_index::File::write_from_index_paths()].. + /// The error returned by [`multi_index::File::write_from_index_paths()`][super::multi_index::File::write_from_index_paths()].. #[derive(Debug, thiserror::Error)] #[allow(missing_docs)] pub enum Error { |