summaryrefslogtreecommitdiffstats
path: root/vendor/ruzstd/src
diff options
context:
space:
mode:
authorDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-30 03:59:35 +0000
committerDaniel Baumann <daniel.baumann@progress-linux.org>2024-05-30 03:59:35 +0000
commitd1b2d29528b7794b41e66fc2136e395a02f8529b (patch)
treea4a17504b260206dec3cf55b2dca82929a348ac2 /vendor/ruzstd/src
parentReleasing progress-linux version 1.72.1+dfsg1-1~progress7.99u1. (diff)
downloadrustc-d1b2d29528b7794b41e66fc2136e395a02f8529b.tar.xz
rustc-d1b2d29528b7794b41e66fc2136e395a02f8529b.zip
Merging upstream version 1.73.0+dfsg1.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'vendor/ruzstd/src')
-rw-r--r--vendor/ruzstd/src/blocks/block.rs4
-rw-r--r--vendor/ruzstd/src/blocks/literals_section.rs4
-rw-r--r--vendor/ruzstd/src/blocks/sequence_section.rs4
-rw-r--r--vendor/ruzstd/src/decoding/block_decoder.rs42
-rw-r--r--vendor/ruzstd/src/decoding/decodebuffer.rs41
-rw-r--r--vendor/ruzstd/src/decoding/dictionary.rs3
-rw-r--r--vendor/ruzstd/src/decoding/literals_section_decoder.rs5
-rw-r--r--vendor/ruzstd/src/decoding/ringbuffer.rs19
-rw-r--r--vendor/ruzstd/src/decoding/scratch.rs38
-rw-r--r--vendor/ruzstd/src/decoding/sequence_execution.rs2
-rw-r--r--vendor/ruzstd/src/decoding/sequence_section_decoder.rs68
-rw-r--r--vendor/ruzstd/src/frame.rs160
-rw-r--r--vendor/ruzstd/src/frame_decoder.rs173
-rw-r--r--vendor/ruzstd/src/fse/fse_decoder.rs11
-rw-r--r--vendor/ruzstd/src/huff0/huff0_decoder.rs22
-rw-r--r--vendor/ruzstd/src/io.rs2
-rw-r--r--vendor/ruzstd/src/io_nostd.rs152
-rw-r--r--vendor/ruzstd/src/lib.rs29
-rw-r--r--vendor/ruzstd/src/streaming_decoder.rs47
-rw-r--r--vendor/ruzstd/src/tests/decode_corpus.rs5
-rw-r--r--vendor/ruzstd/src/tests/dict_test.rs9
-rw-r--r--vendor/ruzstd/src/tests/fuzz_regressions.rs2
-rw-r--r--vendor/ruzstd/src/tests/mod.rs137
23 files changed, 590 insertions, 389 deletions
diff --git a/vendor/ruzstd/src/blocks/block.rs b/vendor/ruzstd/src/blocks/block.rs
index 9c872ebf9..078eb44ee 100644
--- a/vendor/ruzstd/src/blocks/block.rs
+++ b/vendor/ruzstd/src/blocks/block.rs
@@ -6,8 +6,8 @@ pub enum BlockType {
Reserved,
}
-impl std::fmt::Display for BlockType {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+impl core::fmt::Display for BlockType {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> {
match self {
BlockType::Compressed => write!(f, "Compressed"),
BlockType::Raw => write!(f, "Raw"),
diff --git a/vendor/ruzstd/src/blocks/literals_section.rs b/vendor/ruzstd/src/blocks/literals_section.rs
index 9d8307223..159e3b303 100644
--- a/vendor/ruzstd/src/blocks/literals_section.rs
+++ b/vendor/ruzstd/src/blocks/literals_section.rs
@@ -25,8 +25,8 @@ pub enum LiteralsSectionParseError {
NotEnoughBytes { have: usize, need: u8 },
}
-impl std::fmt::Display for LiteralsSectionType {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+impl core::fmt::Display for LiteralsSectionType {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> {
match self {
LiteralsSectionType::Compressed => write!(f, "Compressed"),
LiteralsSectionType::Raw => write!(f, "Raw"),
diff --git a/vendor/ruzstd/src/blocks/sequence_section.rs b/vendor/ruzstd/src/blocks/sequence_section.rs
index 28b9efff2..544755e03 100644
--- a/vendor/ruzstd/src/blocks/sequence_section.rs
+++ b/vendor/ruzstd/src/blocks/sequence_section.rs
@@ -10,8 +10,8 @@ pub struct Sequence {
pub of: u32,
}
-impl std::fmt::Display for Sequence {
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> {
+impl core::fmt::Display for Sequence {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> {
write!(f, "LL: {}, ML: {}, OF: {}", self.ll, self.ml, self.of)
}
}
diff --git a/vendor/ruzstd/src/decoding/block_decoder.rs b/vendor/ruzstd/src/decoding/block_decoder.rs
index 11a4c28c1..0fbce8f48 100644
--- a/vendor/ruzstd/src/decoding/block_decoder.rs
+++ b/vendor/ruzstd/src/decoding/block_decoder.rs
@@ -11,7 +11,7 @@ use crate::blocks::literals_section::LiteralsSectionParseError;
use crate::blocks::sequence_section::SequencesHeaderParseError;
use crate::decoding::scratch::DecoderScratch;
use crate::decoding::sequence_execution::execute_sequences;
-use std::io::{self, Read};
+use crate::io::{self, Read};
pub struct BlockDecoder {
header_buffer: [u8; 3],
@@ -203,12 +203,12 @@ impl BlockDecoder {
let mut section = LiteralsSection::new();
let bytes_in_literals_header = section.parse_from_header(raw)?;
let raw = &raw[bytes_in_literals_header as usize..];
- if crate::VERBOSE {
- println!(
- "Found {} literalssection with regenerated size: {}, and compressed size: {:?}",
- section.ls_type, section.regenerated_size, section.compressed_size
- );
- }
+ vprintln!(
+ "Found {} literalssection with regenerated size: {}, and compressed size: {:?}",
+ section.ls_type,
+ section.regenerated_size,
+ section.compressed_size
+ );
let upper_limit_for_literals = match section.compressed_size {
Some(x) => x as usize,
@@ -227,9 +227,7 @@ impl BlockDecoder {
}
let raw_literals = &raw[..upper_limit_for_literals];
- if crate::VERBOSE {
- println!("Slice for literals: {}", raw_literals.len());
- }
+ vprintln!("Slice for literals: {}", raw_literals.len());
workspace.literals_buffer.clear(); //all literals of the previous block must have been used in the sequence execution anyways. just be defensive here
let bytes_used_in_literals_section = decode_literals(
@@ -247,20 +245,16 @@ impl BlockDecoder {
assert!(bytes_used_in_literals_section == upper_limit_for_literals as u32);
let raw = &raw[upper_limit_for_literals..];
- if crate::VERBOSE {
- println!("Slice for sequences with headers: {}", raw.len());
- }
+ vprintln!("Slice for sequences with headers: {}", raw.len());
let mut seq_section = SequencesHeader::new();
let bytes_in_sequence_header = seq_section.parse_from_header(raw)?;
let raw = &raw[bytes_in_sequence_header as usize..];
- if crate::VERBOSE {
- println!(
- "Found sequencessection with sequences: {} and size: {}",
- seq_section.num_sequences,
- raw.len()
- );
- }
+ vprintln!(
+ "Found sequencessection with sequences: {} and size: {}",
+ seq_section.num_sequences,
+ raw.len()
+ );
assert!(
u32::from(bytes_in_literals_header)
@@ -269,9 +263,7 @@ impl BlockDecoder {
+ raw.len() as u32
== header.content_size
);
- if crate::VERBOSE {
- println!("Slice for sequences: {}", raw.len());
- }
+ vprintln!("Slice for sequences: {}", raw.len());
if seq_section.num_sequences != 0 {
decode_sequences(
@@ -280,9 +272,7 @@ impl BlockDecoder {
&mut workspace.fse,
&mut workspace.sequences,
)?;
- if crate::VERBOSE {
- println!("Executing sequences");
- }
+ vprintln!("Executing sequences");
execute_sequences(workspace)?;
} else {
workspace.buffer.push(&workspace.literals_buffer);
diff --git a/vendor/ruzstd/src/decoding/decodebuffer.rs b/vendor/ruzstd/src/decoding/decodebuffer.rs
index 0dea7015e..33cc58c30 100644
--- a/vendor/ruzstd/src/decoding/decodebuffer.rs
+++ b/vendor/ruzstd/src/decoding/decodebuffer.rs
@@ -1,5 +1,6 @@
-use std::hash::Hasher;
-use std::io;
+use crate::io::{Error, Read, Write};
+use alloc::vec::Vec;
+use core::hash::Hasher;
use twox_hash::XxHash64;
@@ -23,8 +24,8 @@ pub enum DecodebufferError {
OffsetTooBig { offset: usize, buf_len: usize },
}
-impl io::Read for Decodebuffer {
- fn read(&mut self, target: &mut [u8]) -> io::Result<usize> {
+impl Read for Decodebuffer {
+ fn read(&mut self, target: &mut [u8]) -> Result<usize, Error> {
let max_amount = self.can_drain_to_window_size().unwrap_or(0);
let amount = max_amount.min(target.len());
@@ -176,7 +177,7 @@ impl Decodebuffer {
}
}
- pub fn drain_to_window_size_writer(&mut self, mut sink: impl io::Write) -> io::Result<usize> {
+ pub fn drain_to_window_size_writer(&mut self, mut sink: impl Write) -> Result<usize, Error> {
match self.can_drain_to_window_size() {
None => Ok(0),
Some(can_drain) => {
@@ -199,14 +200,14 @@ impl Decodebuffer {
vec
}
- pub fn drain_to_writer(&mut self, mut sink: impl io::Write) -> io::Result<usize> {
+ pub fn drain_to_writer(&mut self, mut sink: impl Write) -> Result<usize, Error> {
let len = self.buffer.len();
self.drain_to(len, |buf| write_all_bytes(&mut sink, buf))?;
Ok(len)
}
- pub fn read_all(&mut self, target: &mut [u8]) -> io::Result<usize> {
+ pub fn read_all(&mut self, target: &mut [u8]) -> Result<usize, Error> {
let amount = self.buffer.len().min(target.len());
let mut written = 0;
@@ -224,8 +225,8 @@ impl Decodebuffer {
fn drain_to(
&mut self,
amount: usize,
- mut write_bytes: impl FnMut(&[u8]) -> (usize, io::Result<()>),
- ) -> io::Result<()> {
+ mut write_bytes: impl FnMut(&[u8]) -> (usize, Result<(), Error>),
+ ) -> Result<(), Error> {
if amount == 0 {
return Ok(());
}
@@ -280,7 +281,7 @@ impl Decodebuffer {
}
/// Like Write::write_all but returns partial write length even on error
-fn write_all_bytes(mut sink: impl io::Write, buf: &[u8]) -> (usize, io::Result<()>) {
+fn write_all_bytes(mut sink: impl Write, buf: &[u8]) -> (usize, Result<(), Error>) {
let mut written = 0;
while written < buf.len() {
match sink.write(&buf[written..]) {
@@ -294,7 +295,11 @@ fn write_all_bytes(mut sink: impl io::Write, buf: &[u8]) -> (usize, io::Result<(
#[cfg(test)]
mod tests {
use super::Decodebuffer;
- use std::io::Write;
+ use crate::io::{Error, ErrorKind, Write};
+
+ extern crate std;
+ use alloc::vec;
+ use alloc::vec::Vec;
#[test]
fn short_writer() {
@@ -304,7 +309,7 @@ mod tests {
}
impl Write for ShortWriter {
- fn write(&mut self, buf: &[u8]) -> std::result::Result<usize, std::io::Error> {
+ fn write(&mut self, buf: &[u8]) -> std::result::Result<usize, Error> {
if buf.len() > self.write_len {
self.buf.extend_from_slice(&buf[..self.write_len]);
Ok(self.write_len)
@@ -314,7 +319,7 @@ mod tests {
}
}
- fn flush(&mut self) -> std::result::Result<(), std::io::Error> {
+ fn flush(&mut self) -> std::result::Result<(), Error> {
Ok(())
}
}
@@ -352,18 +357,18 @@ mod tests {
}
impl Write for WouldblockWriter {
- fn write(&mut self, buf: &[u8]) -> std::result::Result<usize, std::io::Error> {
+ fn write(&mut self, buf: &[u8]) -> std::result::Result<usize, Error> {
if self.last_blocked < self.block_every {
self.buf.extend_from_slice(buf);
self.last_blocked += 1;
Ok(buf.len())
} else {
self.last_blocked = 0;
- Err(std::io::Error::from(std::io::ErrorKind::WouldBlock))
+ Err(Error::from(ErrorKind::WouldBlock))
}
}
- fn flush(&mut self) -> std::result::Result<(), std::io::Error> {
+ fn flush(&mut self) -> std::result::Result<(), Error> {
Ok(())
}
}
@@ -390,7 +395,7 @@ mod tests {
}
}
Err(e) => {
- if e.kind() == std::io::ErrorKind::WouldBlock {
+ if e.kind() == ErrorKind::WouldBlock {
continue;
} else {
panic!("Unexpected error {:?}", e);
@@ -410,7 +415,7 @@ mod tests {
}
}
Err(e) => {
- if e.kind() == std::io::ErrorKind::WouldBlock {
+ if e.kind() == ErrorKind::WouldBlock {
continue;
} else {
panic!("Unexpected error {:?}", e);
diff --git a/vendor/ruzstd/src/decoding/dictionary.rs b/vendor/ruzstd/src/decoding/dictionary.rs
index 51fbcdf0b..aa6769371 100644
--- a/vendor/ruzstd/src/decoding/dictionary.rs
+++ b/vendor/ruzstd/src/decoding/dictionary.rs
@@ -1,4 +1,5 @@
-use std::convert::TryInto;
+use alloc::vec::Vec;
+use core::convert::TryInto;
use crate::decoding::scratch::FSEScratch;
use crate::decoding::scratch::HuffmanScratch;
diff --git a/vendor/ruzstd/src/decoding/literals_section_decoder.rs b/vendor/ruzstd/src/decoding/literals_section_decoder.rs
index d947f87eb..bd7fb18ea 100644
--- a/vendor/ruzstd/src/decoding/literals_section_decoder.rs
+++ b/vendor/ruzstd/src/decoding/literals_section_decoder.rs
@@ -2,6 +2,7 @@ use super::super::blocks::literals_section::{LiteralsSection, LiteralsSectionTyp
use super::bit_reader_reverse::{BitReaderReversed, GetBitsError};
use super::scratch::HuffmanScratch;
use crate::huff0::{HuffmanDecoder, HuffmanDecoderError, HuffmanTableError};
+use alloc::vec::Vec;
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
@@ -75,9 +76,7 @@ fn decompress_literals(
LiteralsSectionType::Compressed => {
//read Huffman tree description
bytes_read += scratch.table.build_decoder(source)?;
- if crate::VERBOSE {
- println!("Built huffman table using {} bytes", bytes_read);
- }
+ vprintln!("Built huffman table using {} bytes", bytes_read);
}
LiteralsSectionType::Treeless => {
if scratch.table.max_num_bits == 0 {
diff --git a/vendor/ruzstd/src/decoding/ringbuffer.rs b/vendor/ruzstd/src/decoding/ringbuffer.rs
index 9e3e9ba5e..fc9a2e330 100644
--- a/vendor/ruzstd/src/decoding/ringbuffer.rs
+++ b/vendor/ruzstd/src/decoding/ringbuffer.rs
@@ -1,4 +1,5 @@
-use std::{alloc::Layout, ptr::NonNull, slice};
+use alloc::alloc::{alloc, dealloc};
+use core::{alloc::Layout, ptr::NonNull, slice};
pub struct RingBuffer {
buf: NonNull<u8>,
@@ -70,7 +71,7 @@ impl RingBuffer {
// alloc the new memory region and panic if alloc fails
// TODO maybe rework this to generate an error?
let new_buf = unsafe {
- let new_buf = std::alloc::alloc(new_layout);
+ let new_buf = alloc(new_layout);
NonNull::new(new_buf).expect("Allocating new space for the ringbuffer failed")
};
@@ -85,7 +86,7 @@ impl RingBuffer {
.as_ptr()
.add(s1_len)
.copy_from_nonoverlapping(s2_ptr, s2_len);
- std::alloc::dealloc(self.buf.as_ptr(), current_layout);
+ dealloc(self.buf.as_ptr(), current_layout);
}
self.tail = s1_len + s2_len;
@@ -341,7 +342,7 @@ impl Drop for RingBuffer {
let current_layout = unsafe { Layout::array::<u8>(self.cap).unwrap_unchecked() };
unsafe {
- std::alloc::dealloc(self.buf.as_ptr(), current_layout);
+ dealloc(self.buf.as_ptr(), current_layout);
}
}
}
@@ -448,8 +449,8 @@ unsafe fn copy_with_nobranch_check(
f1_ptr.copy_from_nonoverlapping(m1_ptr, m1_in_f1);
f2_ptr.copy_from_nonoverlapping(m1_ptr.add(m1_in_f1), m1_in_f2);
}
- 6 => std::hint::unreachable_unchecked(),
- 7 => std::hint::unreachable_unchecked(),
+ 6 => core::hint::unreachable_unchecked(),
+ 7 => core::hint::unreachable_unchecked(),
9 => {
f1_ptr.copy_from_nonoverlapping(m1_ptr, m1_in_f1);
f2_ptr.copy_from_nonoverlapping(m2_ptr, m2_in_f2);
@@ -480,9 +481,9 @@ unsafe fn copy_with_nobranch_check(
.add(m1_in_f2)
.copy_from_nonoverlapping(m2_ptr, m2_in_f2);
}
- 14 => std::hint::unreachable_unchecked(),
- 15 => std::hint::unreachable_unchecked(),
- _ => std::hint::unreachable_unchecked(),
+ 14 => core::hint::unreachable_unchecked(),
+ 15 => core::hint::unreachable_unchecked(),
+ _ => core::hint::unreachable_unchecked(),
}
}
diff --git a/vendor/ruzstd/src/decoding/scratch.rs b/vendor/ruzstd/src/decoding/scratch.rs
index 2bd753bde..35d5c61ef 100644
--- a/vendor/ruzstd/src/decoding/scratch.rs
+++ b/vendor/ruzstd/src/decoding/scratch.rs
@@ -3,6 +3,7 @@ use super::decodebuffer::Decodebuffer;
use crate::decoding::dictionary::Dictionary;
use crate::fse::FSETable;
use crate::huff0::HuffmanTable;
+use alloc::vec::Vec;
pub struct DecoderScratch {
pub huf: HuffmanScratch,
@@ -56,30 +57,17 @@ impl DecoderScratch {
self.huf.table.reset();
}
- pub fn use_dict(&mut self, dict: &Dictionary) {
- self.fse = dict.fse.clone();
- self.huf = dict.huf.clone();
+ pub fn init_from_dict(&mut self, dict: &Dictionary) {
+ self.fse.reinit_from(&dict.fse);
+ self.huf.table.reinit_from(&dict.huf.table);
self.offset_hist = dict.offset_hist;
- self.buffer.dict_content = dict.dict_content.clone();
- }
-
- /// parses the dictionary and set the tables
- /// it returns the dict_id for checking with the frame's dict_id
- pub fn load_dict(
- &mut self,
- raw: &[u8],
- ) -> Result<u32, super::dictionary::DictionaryDecodeError> {
- let dict = super::dictionary::Dictionary::decode_dict(raw)?;
-
- self.huf = dict.huf.clone();
- self.fse = dict.fse.clone();
- self.offset_hist = dict.offset_hist;
- self.buffer.dict_content = dict.dict_content.clone();
- Ok(dict.id)
+ self.buffer.dict_content.clear();
+ self.buffer
+ .dict_content
+ .extend_from_slice(&dict.dict_content);
}
}
-#[derive(Clone)]
pub struct HuffmanScratch {
pub table: HuffmanTable,
}
@@ -98,7 +86,6 @@ impl Default for HuffmanScratch {
}
}
-#[derive(Clone)]
pub struct FSEScratch {
pub offsets: FSETable,
pub of_rle: Option<u8>,
@@ -119,6 +106,15 @@ impl FSEScratch {
ml_rle: None,
}
}
+
+ pub fn reinit_from(&mut self, other: &Self) {
+ self.offsets.reinit_from(&other.offsets);
+ self.literal_lengths.reinit_from(&other.literal_lengths);
+ self.match_lengths.reinit_from(&other.match_lengths);
+ self.of_rle = other.of_rle;
+ self.ll_rle = other.ll_rle;
+ self.ml_rle = other.ml_rle;
+ }
}
impl Default for FSEScratch {
diff --git a/vendor/ruzstd/src/decoding/sequence_execution.rs b/vendor/ruzstd/src/decoding/sequence_execution.rs
index 19247ec62..5946df18c 100644
--- a/vendor/ruzstd/src/decoding/sequence_execution.rs
+++ b/vendor/ruzstd/src/decoding/sequence_execution.rs
@@ -18,8 +18,6 @@ pub fn execute_sequences(scratch: &mut DecoderScratch) -> Result<(), ExecuteSequ
for idx in 0..scratch.sequences.len() {
let seq = scratch.sequences[idx];
- if crate::VERBOSE {}
- //println!("{}: {}", idx, seq);
if seq.ll > 0 {
let high = literals_copy_counter + seq.ll as usize;
diff --git a/vendor/ruzstd/src/decoding/sequence_section_decoder.rs b/vendor/ruzstd/src/decoding/sequence_section_decoder.rs
index 3d5990c05..6c366fba1 100644
--- a/vendor/ruzstd/src/decoding/sequence_section_decoder.rs
+++ b/vendor/ruzstd/src/decoding/sequence_section_decoder.rs
@@ -4,6 +4,7 @@ use super::super::blocks::sequence_section::SequencesHeader;
use super::bit_reader_reverse::{BitReaderReversed, GetBitsError};
use super::scratch::FSEScratch;
use crate::fse::{FSEDecoder, FSEDecoderError, FSETableError};
+use alloc::vec::Vec;
#[derive(Debug, thiserror::Error)]
#[non_exhaustive]
@@ -42,9 +43,7 @@ pub fn decode_sequences(
) -> Result<(), DecodeSequenceError> {
let bytes_read = maybe_update_fse_tables(section, source, scratch)?;
- if crate::VERBOSE {
- println!("Updating tables used {} bytes", bytes_read);
- }
+ vprintln!("Updating tables used {} bytes", bytes_read);
let bit_stream = &source[bytes_read..];
@@ -319,16 +318,13 @@ fn maybe_update_fse_tables(
ModeType::FSECompressed => {
let bytes = scratch.literal_lengths.build_decoder(source, LL_MAX_LOG)?;
bytes_read += bytes;
- if crate::VERBOSE {
- println!("Updating ll table");
- println!("Used bytes: {}", bytes);
- }
+
+ vprintln!("Updating ll table");
+ vprintln!("Used bytes: {}", bytes);
scratch.ll_rle = None;
}
ModeType::RLE => {
- if crate::VERBOSE {
- println!("Use RLE ll table");
- }
+ vprintln!("Use RLE ll table");
if source.is_empty() {
return Err(DecodeSequenceError::MissingByteForRleLlTable);
}
@@ -336,9 +332,7 @@ fn maybe_update_fse_tables(
scratch.ll_rle = Some(source[0]);
}
ModeType::Predefined => {
- if crate::VERBOSE {
- println!("Use predefined ll table");
- }
+ vprintln!("Use predefined ll table");
scratch.literal_lengths.build_from_probabilities(
LL_DEFAULT_ACC_LOG,
&Vec::from(&LITERALS_LENGTH_DEFAULT_DISTRIBUTION[..]),
@@ -346,9 +340,7 @@ fn maybe_update_fse_tables(
scratch.ll_rle = None;
}
ModeType::Repeat => {
- if crate::VERBOSE {
- println!("Repeat ll table");
- }
+ vprintln!("Repeat ll table");
/* Nothing to do */
}
};
@@ -358,17 +350,13 @@ fn maybe_update_fse_tables(
match modes.of_mode() {
ModeType::FSECompressed => {
let bytes = scratch.offsets.build_decoder(of_source, OF_MAX_LOG)?;
- if crate::VERBOSE {
- println!("Updating of table");
- println!("Used bytes: {}", bytes);
- }
+ vprintln!("Updating of table");
+ vprintln!("Used bytes: {}", bytes);
bytes_read += bytes;
scratch.of_rle = None;
}
ModeType::RLE => {
- if crate::VERBOSE {
- println!("Use RLE of table");
- }
+ vprintln!("Use RLE of table");
if of_source.is_empty() {
return Err(DecodeSequenceError::MissingByteForRleOfTable);
}
@@ -376,9 +364,7 @@ fn maybe_update_fse_tables(
scratch.of_rle = Some(of_source[0]);
}
ModeType::Predefined => {
- if crate::VERBOSE {
- println!("Use predefined of table");
- }
+ vprintln!("Use predefined of table");
scratch.offsets.build_from_probabilities(
OF_DEFAULT_ACC_LOG,
&Vec::from(&OFFSET_DEFAULT_DISTRIBUTION[..]),
@@ -386,9 +372,7 @@ fn maybe_update_fse_tables(
scratch.of_rle = None;
}
ModeType::Repeat => {
- if crate::VERBOSE {
- println!("Repeat of table");
- }
+ vprintln!("Repeat of table");
/* Nothing to do */
}
};
@@ -399,16 +383,12 @@ fn maybe_update_fse_tables(
ModeType::FSECompressed => {
let bytes = scratch.match_lengths.build_decoder(ml_source, ML_MAX_LOG)?;
bytes_read += bytes;
- if crate::VERBOSE {
- println!("Updating ml table");
- println!("Used bytes: {}", bytes);
- }
+ vprintln!("Updating ml table");
+ vprintln!("Used bytes: {}", bytes);
scratch.ml_rle = None;
}
ModeType::RLE => {
- if crate::VERBOSE {
- println!("Use RLE ml table");
- }
+ vprintln!("Use RLE ml table");
if ml_source.is_empty() {
return Err(DecodeSequenceError::MissingByteForRleMlTable);
}
@@ -416,9 +396,7 @@ fn maybe_update_fse_tables(
scratch.ml_rle = Some(ml_source[0]);
}
ModeType::Predefined => {
- if crate::VERBOSE {
- println!("Use predefined ml table");
- }
+ vprintln!("Use predefined ml table");
scratch.match_lengths.build_from_probabilities(
ML_DEFAULT_ACC_LOG,
&Vec::from(&MATCH_LENGTH_DEFAULT_DISTRIBUTION[..]),
@@ -426,9 +404,7 @@ fn maybe_update_fse_tables(
scratch.ml_rle = None;
}
ModeType::Repeat => {
- if crate::VERBOSE {
- println!("Repeat ml table");
- }
+ vprintln!("Repeat ml table");
/* Nothing to do */
}
};
@@ -463,10 +439,14 @@ fn test_ll_default() {
)
.unwrap();
+ #[cfg(feature = "std")]
for idx in 0..table.decode.len() {
- println!(
+ std::println!(
"{:3}: {:3} {:3} {:3}",
- idx, table.decode[idx].symbol, table.decode[idx].num_bits, table.decode[idx].base_line
+ idx,
+ table.decode[idx].symbol,
+ table.decode[idx].num_bits,
+ table.decode[idx].base_line
);
}
diff --git a/vendor/ruzstd/src/frame.rs b/vendor/ruzstd/src/frame.rs
index 1eb3f8911..3810e4bca 100644
--- a/vendor/ruzstd/src/frame.rs
+++ b/vendor/ruzstd/src/frame.rs
@@ -1,20 +1,17 @@
-use std::convert::TryInto;
-use std::io;
-
+use crate::io::{Error, Read};
pub const MAGIC_NUM: u32 = 0xFD2F_B528;
pub const MIN_WINDOW_SIZE: u64 = 1024;
pub const MAX_WINDOW_SIZE: u64 = (1 << 41) + 7 * (1 << 38);
pub struct Frame {
- magic_num: u32,
pub header: FrameHeader,
}
pub struct FrameHeader {
pub descriptor: FrameDescriptor,
window_descriptor: u8,
- dict_id: Vec<u8>,
- frame_content_size: Vec<u8>,
+ dict_id: Option<u32>,
+ frame_content_size: u64,
}
pub struct FrameDescriptor(u8);
@@ -97,7 +94,7 @@ pub enum FrameHeaderError {
impl FrameHeader {
pub fn window_size(&self) -> Result<u64, FrameHeaderError> {
if self.descriptor.single_segment_flag() {
- self.frame_content_size()
+ Ok(self.frame_content_size())
} else {
let exp = self.window_descriptor >> 3;
let mantissa = self.window_descriptor & 0x7;
@@ -120,93 +117,12 @@ impl FrameHeader {
}
}
- pub fn dictionary_id(&self) -> Result<Option<u32>, FrameHeaderError> {
- if self.descriptor.dict_id_flag() == 0 {
- Ok(None)
- } else {
- let bytes = self.descriptor.dictionary_id_bytes()?;
- if self.dict_id.len() != bytes as usize {
- Err(FrameHeaderError::DictIdTooSmall {
- got: self.dict_id.len(),
- expected: bytes as usize,
- })
- } else {
- let mut value: u32 = 0;
- let mut shift = 0;
- for x in &self.dict_id {
- value |= u32::from(*x) << shift;
- shift += 8;
- }
-
- Ok(Some(value))
- }
- }
+ pub fn dictionary_id(&self) -> Option<u32> {
+ self.dict_id
}
- pub fn frame_content_size(&self) -> Result<u64, FrameHeaderError> {
- let bytes = self.descriptor.frame_content_size_bytes()?;
-
- if self.frame_content_size.len() != (bytes as usize) {
- return Err(FrameHeaderError::MismatchedFrameSize {
- got: self.frame_content_size.len(),
- expected: bytes,
- });
- }
-
- match bytes {
- 0 => Err(FrameHeaderError::FrameSizeIsZero),
- 1 => Ok(u64::from(self.frame_content_size[0])),
- 2 => {
- let val = (u64::from(self.frame_content_size[1]) << 8)
- + (u64::from(self.frame_content_size[0]));
- Ok(val + 256) //this weird offset is from the documentation. Only if bytes == 2
- }
- 4 => {
- let val = self.frame_content_size[..4]
- .try_into()
- .expect("optimized away");
- let val = u32::from_le_bytes(val);
- Ok(u64::from(val))
- }
- 8 => {
- let val = self.frame_content_size[..8]
- .try_into()
- .expect("optimized away");
- let val = u64::from_le_bytes(val);
- Ok(val)
- }
- other => Err(FrameHeaderError::InvalidFrameSize { got: other }),
- }
- }
-}
-
-#[derive(Debug, thiserror::Error)]
-#[non_exhaustive]
-pub enum FrameCheckError {
- #[error("magic_num wrong. Is: {got}. Should be: {MAGIC_NUM}")]
- WrongMagicNum { got: u32 },
- #[error("Reserved Flag set. Must be zero")]
- ReservedFlagSet,
- #[error(transparent)]
- FrameHeaderError(#[from] FrameHeaderError),
-}
-
-impl Frame {
- pub fn check_valid(&self) -> Result<(), FrameCheckError> {
- if self.magic_num != MAGIC_NUM {
- Err(FrameCheckError::WrongMagicNum {
- got: self.magic_num,
- })
- } else if self.header.descriptor.reserved_flag() {
- Err(FrameCheckError::ReservedFlagSet)
- } else {
- self.header.dictionary_id()?;
- self.header.window_size()?;
- if self.header.descriptor.single_segment_flag() {
- self.header.frame_content_size()?;
- }
- Ok(())
- }
+ pub fn frame_content_size(&self) -> u64 {
+ self.frame_content_size
}
}
@@ -214,27 +130,29 @@ impl Frame {
#[non_exhaustive]
pub enum ReadFrameHeaderError {
#[error("Error while reading magic number: {0}")]
- MagicNumberReadError(#[source] io::Error),
+ MagicNumberReadError(#[source] Error),
+ #[error("Read wrong magic number: 0x{0:X}")]
+ BadMagicNumber(u32),
#[error("Error while reading frame descriptor: {0}")]
- FrameDescriptorReadError(#[source] io::Error),
+ FrameDescriptorReadError(#[source] Error),
#[error(transparent)]
InvalidFrameDescriptor(#[from] FrameDescriptorError),
#[error("Error while reading window descriptor: {0}")]
- WindowDescriptorReadError(#[source] io::Error),
+ WindowDescriptorReadError(#[source] Error),
#[error("Error while reading dictionary id: {0}")]
- DictionaryIdReadError(#[source] io::Error),
+ DictionaryIdReadError(#[source] Error),
#[error("Error while reading frame content size: {0}")]
- FrameContentSizeReadError(#[source] io::Error),
+ FrameContentSizeReadError(#[source] Error),
#[error("SkippableFrame encountered with MagicNumber 0x{0:X} and length {1} bytes")]
SkipFrame(u32, u32),
}
-use std::io::Read;
pub fn read_frame_header(mut r: impl Read) -> Result<(Frame, u8), ReadFrameHeaderError> {
use ReadFrameHeaderError as err;
let mut buf = [0u8; 4];
r.read_exact(&mut buf).map_err(err::MagicNumberReadError)?;
+ let mut bytes_read = 4;
let magic_num = u32::from_le_bytes(buf);
// Skippable frames have a magic number in this interval
@@ -245,7 +163,9 @@ pub fn read_frame_header(mut r: impl Read) -> Result<(Frame, u8), ReadFrameHeade
return Err(ReadFrameHeaderError::SkipFrame(magic_num, skip_size));
}
- let mut bytes_read = 4;
+ if magic_num != MAGIC_NUM {
+ return Err(ReadFrameHeaderError::BadMagicNumber(magic_num));
+ }
r.read_exact(&mut buf[0..1])
.map_err(err::FrameDescriptorReadError)?;
@@ -255,8 +175,8 @@ pub fn read_frame_header(mut r: impl Read) -> Result<(Frame, u8), ReadFrameHeade
let mut frame_header = FrameHeader {
descriptor: FrameDescriptor(desc.0),
- dict_id: vec![0; desc.dictionary_id_bytes()? as usize],
- frame_content_size: vec![0; desc.frame_content_size_bytes()? as usize],
+ dict_id: None,
+ frame_content_size: 0,
window_descriptor: 0,
};
@@ -267,20 +187,42 @@ pub fn read_frame_header(mut r: impl Read) -> Result<(Frame, u8), ReadFrameHeade
bytes_read += 1;
}
- if !frame_header.dict_id.is_empty() {
- r.read_exact(frame_header.dict_id.as_mut_slice())
- .map_err(err::DictionaryIdReadError)?;
- bytes_read += frame_header.dict_id.len();
+ let dict_id_len = desc.dictionary_id_bytes()? as usize;
+ if dict_id_len != 0 {
+ let buf = &mut buf[..dict_id_len];
+ r.read_exact(buf).map_err(err::DictionaryIdReadError)?;
+ bytes_read += dict_id_len;
+ let mut dict_id = 0u32;
+
+ #[allow(clippy::needless_range_loop)]
+ for i in 0..dict_id_len {
+ dict_id += (buf[i] as u32) << (8 * i);
+ }
+ if dict_id != 0 {
+ frame_header.dict_id = Some(dict_id);
+ }
}
- if !frame_header.frame_content_size.is_empty() {
- r.read_exact(frame_header.frame_content_size.as_mut_slice())
+ let fcs_len = desc.frame_content_size_bytes()? as usize;
+ if fcs_len != 0 {
+ let mut fcs_buf = [0u8; 8];
+ let fcs_buf = &mut fcs_buf[..fcs_len];
+ r.read_exact(fcs_buf)
.map_err(err::FrameContentSizeReadError)?;
- bytes_read += frame_header.frame_content_size.len();
+ bytes_read += fcs_len;
+ let mut fcs = 0u64;
+
+ #[allow(clippy::needless_range_loop)]
+ for i in 0..fcs_len {
+ fcs += (fcs_buf[i] as u64) << (8 * i);
+ }
+ if fcs_len == 2 {
+ fcs += 256;
+ }
+ frame_header.frame_content_size = fcs;
}
let frame: Frame = Frame {
- magic_num,
header: frame_header,
};
diff --git a/vendor/ruzstd/src/frame_decoder.rs b/vendor/ruzstd/src/frame_decoder.rs
index 560e82810..18af6b3d3 100644
--- a/vendor/ruzstd/src/frame_decoder.rs
+++ b/vendor/ruzstd/src/frame_decoder.rs
@@ -2,10 +2,11 @@ use super::frame;
use crate::decoding::dictionary::Dictionary;
use crate::decoding::scratch::DecoderScratch;
use crate::decoding::{self, dictionary};
-use std::collections::HashMap;
-use std::convert::TryInto;
-use std::hash::Hasher;
-use std::io::{self, Read};
+use crate::io::{Error, Read, Write};
+use alloc::collections::BTreeMap;
+use alloc::vec::Vec;
+use core::convert::TryInto;
+use core::hash::Hasher;
/// This implements a decoder for zstd frames. This decoder is able to decode frames only partially and gives control
/// over how many bytes/blocks will be decoded at a time (so you don't have to decode a 10GB file into memory all at once).
@@ -17,11 +18,15 @@ use std::io::{self, Read};
/// Workflow is as follows:
/// ```
/// use ruzstd::frame_decoder::BlockDecodingStrategy;
-/// use std::io::Read;
-/// use std::io::Write;
///
+/// # #[cfg(feature = "std")]
+/// use std::io::{Read, Write};
///
-/// fn decode_this(mut file: impl std::io::Read) {
+/// // no_std environments can use the crate's own Read traits
+/// # #[cfg(not(feature = "std"))]
+/// use ruzstd::io::{Read, Write};
+///
+/// fn decode_this(mut file: impl Read) {
/// //Create a new decoder
/// let mut frame_dec = ruzstd::FrameDecoder::new();
/// let mut result = Vec::new();
@@ -50,12 +55,13 @@ use std::io::{self, Read};
/// }
///
/// fn do_something(data: &[u8]) {
+/// # #[cfg(feature = "std")]
/// std::io::stdout().write_all(data).unwrap();
/// }
/// ```
pub struct FrameDecoder {
state: Option<FrameDecoderState>,
- dicts: HashMap<u32, Dictionary>,
+ dicts: BTreeMap<u32, Dictionary>,
}
struct FrameDecoderState {
@@ -81,8 +87,6 @@ pub enum FrameDecoderError {
ReadFrameHeaderError(#[from] frame::ReadFrameHeaderError),
#[error(transparent)]
FrameHeaderError(#[from] frame::FrameHeaderError),
- #[error(transparent)]
- FrameCheckError(#[from] frame::FrameCheckError),
#[error("Specified window_size is too big; Requested: {requested}, Max: {MAX_WINDOW_SIZE}")]
WindowSizeTooBig { requested: u64 },
#[error(transparent)]
@@ -92,17 +96,17 @@ pub enum FrameDecoderError {
#[error("Failed to parse block header: {0}")]
FailedToReadBlockBody(decoding::block_decoder::DecodeBlockContentError),
#[error("Failed to read checksum: {0}")]
- FailedToReadChecksum(#[source] io::Error),
+ FailedToReadChecksum(#[source] Error),
#[error("Decoder must initialized or reset before using it")]
NotYetInitialized,
#[error("Decoder encountered error while initializing: {0}")]
FailedToInitialize(frame::FrameHeaderError),
#[error("Decoder encountered error while draining the decodebuffer: {0}")]
- FailedToDrainDecodebuffer(#[source] io::Error),
+ FailedToDrainDecodebuffer(#[source] Error),
#[error("Target must have at least as many bytes as the contentsize of the frame reports")]
TargetTooSmall,
- #[error("Frame header specified dictionary id that wasnt provided by add_dict() or reset_with_dict()")]
- DictNotProvided,
+ #[error("Frame header specified dictionary id 0x{dict_id:X} that wasnt provided by add_dict() or reset_with_dict()")]
+ DictNotProvided { dict_id: u32 },
}
const MAX_WINDOW_SIZE: u64 = 1024 * 1024 * 100;
@@ -111,7 +115,6 @@ impl FrameDecoderState {
pub fn new(source: impl Read) -> Result<FrameDecoderState, FrameDecoderError> {
let (frame, header_size) = frame::read_frame_header(source)?;
let window_size = frame.header.window_size()?;
- frame.check_valid()?;
Ok(FrameDecoderState {
frame,
frame_finished: false,
@@ -126,7 +129,6 @@ impl FrameDecoderState {
pub fn reset(&mut self, source: impl Read) -> Result<(), FrameDecoderError> {
let (frame, header_size) = frame::read_frame_header(source)?;
let window_size = frame.header.window_size()?;
- frame.check_valid()?;
if window_size > MAX_WINDOW_SIZE {
return Err(FrameDecoderError::WindowSizeTooBig {
@@ -158,7 +160,7 @@ impl FrameDecoder {
pub fn new() -> FrameDecoder {
FrameDecoder {
state: None,
- dicts: HashMap::new(),
+ dicts: BTreeMap::new(),
}
}
@@ -171,14 +173,6 @@ impl FrameDecoder {
pub fn init(&mut self, source: impl Read) -> Result<(), FrameDecoderError> {
self.reset(source)
}
- /// Like init but provides the dict to use for the next frame
- pub fn init_with_dict(
- &mut self,
- source: impl Read,
- dict: &[u8],
- ) -> Result<(), FrameDecoderError> {
- self.reset_with_dict(source, dict)
- }
/// reset() will allocate all needed buffers if it is the first time this decoder is used
/// else they just reset these buffers with not further allocations
@@ -187,46 +181,55 @@ impl FrameDecoder {
///
/// equivalent to init()
pub fn reset(&mut self, source: impl Read) -> Result<(), FrameDecoderError> {
- match &mut self.state {
- Some(s) => s.reset(source),
+ use FrameDecoderError as err;
+ let state = match &mut self.state {
+ Some(s) => {
+ s.reset(source)?;
+ s
+ }
None => {
self.state = Some(FrameDecoderState::new(source)?);
- Ok(())
+ self.state.as_mut().unwrap()
}
- }
- }
-
- /// Like reset but provides the dict to use for the next frame
- pub fn reset_with_dict(
- &mut self,
- source: impl Read,
- dict: &[u8],
- ) -> Result<(), FrameDecoderError> {
- self.reset(source)?;
- if let Some(state) = &mut self.state {
- let id = state.decoder_scratch.load_dict(dict)?;
- state.using_dict = Some(id);
};
+ if let Some(dict_id) = state.frame.header.dictionary_id() {
+ let dict = self
+ .dicts
+ .get(&dict_id)
+ .ok_or(err::DictNotProvided { dict_id })?;
+ state.decoder_scratch.init_from_dict(dict);
+ state.using_dict = Some(dict_id);
+ }
Ok(())
}
/// Add a dict to the FrameDecoder that can be used when needed. The FrameDecoder uses the appropriate one dynamically
- pub fn add_dict(&mut self, raw_dict: &[u8]) -> Result<(), FrameDecoderError> {
- let dict = Dictionary::decode_dict(raw_dict)?;
+ pub fn add_dict(&mut self, dict: Dictionary) -> Result<(), FrameDecoderError> {
self.dicts.insert(dict.id, dict);
Ok(())
}
- /// Returns how many bytes the frame contains after decompression
- pub fn content_size(&self) -> Option<u64> {
- let state = match &self.state {
- None => return Some(0),
- Some(s) => s,
+ pub fn force_dict(&mut self, dict_id: u32) -> Result<(), FrameDecoderError> {
+ use FrameDecoderError as err;
+ let Some(state) = self.state.as_mut() else {
+ return Err(err::NotYetInitialized);
};
- match state.frame.header.frame_content_size() {
- Err(_) => None,
- Ok(x) => Some(x),
+ let dict = self
+ .dicts
+ .get(&dict_id)
+ .ok_or(err::DictNotProvided { dict_id })?;
+ state.decoder_scratch.init_from_dict(dict);
+ state.using_dict = Some(dict_id);
+
+ Ok(())
+ }
+
+ /// Returns how many bytes the frame contains after decompression
+ pub fn content_size(&self) -> u64 {
+ match &self.state {
+ None => 0,
+ Some(s) => s.frame.header.frame_content_size(),
}
}
@@ -297,47 +300,26 @@ impl FrameDecoder {
use FrameDecoderError as err;
let state = self.state.as_mut().ok_or(err::NotYetInitialized)?;
- if let Some(id) = state.frame.header.dictionary_id().map_err(
- //should never happen we check this directly after decoding the frame header
- err::FailedToInitialize,
- )? {
- match state.using_dict {
- Some(using_id) => {
- //happy
- debug_assert!(id == using_id);
- }
- None => {
- let dict = self.dicts.get(&id).ok_or(err::DictNotProvided)?;
- state.decoder_scratch.use_dict(dict);
- state.using_dict = Some(id);
- }
- }
- }
-
let mut block_dec = decoding::block_decoder::new();
let buffer_size_before = state.decoder_scratch.buffer.len();
let block_counter_before = state.block_counter;
loop {
- if crate::VERBOSE {
- println!("################");
- println!("Next Block: {}", state.block_counter);
- println!("################");
- }
+ vprintln!("################");
+ vprintln!("Next Block: {}", state.block_counter);
+ vprintln!("################");
let (block_header, block_header_size) = block_dec
.read_block_header(&mut source)
.map_err(err::FailedToReadBlockHeader)?;
state.bytes_read_counter += u64::from(block_header_size);
- if crate::VERBOSE {
- println!();
- println!(
- "Found {} block with size: {}, which will be of size: {}",
- block_header.block_type,
- block_header.content_size,
- block_header.decompressed_size
- );
- }
+ vprintln!();
+ vprintln!(
+ "Found {} block with size: {}, which will be of size: {}",
+ block_header.block_type,
+ block_header.content_size,
+ block_header.decompressed_size
+ );
let bytes_read_in_block_body = block_dec
.decode_block_content(&block_header, &mut state.decoder_scratch, &mut source)
@@ -346,9 +328,7 @@ impl FrameDecoder {
state.block_counter += 1;
- if crate::VERBOSE {
- println!("Output: {}", state.decoder_scratch.buffer.len());
- }
+ vprintln!("Output: {}", state.decoder_scratch.buffer.len());
if block_header.last_block {
state.frame_finished = true;
@@ -396,7 +376,7 @@ impl FrameDecoder {
/// Collect bytes and retain window_size bytes while decoding is still going on.
/// After decoding of the frame (is_finished() == true) has finished it will collect all remaining bytes
- pub fn collect_to_writer(&mut self, w: impl std::io::Write) -> Result<usize, std::io::Error> {
+ pub fn collect_to_writer(&mut self, w: impl Write) -> Result<usize, Error> {
let finished = self.is_finished();
let state = match &mut self.state {
None => return Ok(0),
@@ -482,23 +462,6 @@ impl FrameDecoder {
return Ok((4, 0));
}
- if let Some(id) = state.frame.header.dictionary_id().map_err(
- //should never happen we check this directly after decoding the frame header
- err::FailedToInitialize,
- )? {
- match state.using_dict {
- Some(using_id) => {
- //happy
- debug_assert!(id == using_id);
- }
- None => {
- let dict = self.dicts.get(&id).ok_or(err::DictNotProvided)?;
- state.decoder_scratch.use_dict(dict);
- state.using_dict = Some(id);
- }
- }
- }
-
loop {
//check if there are enough bytes for the next header
if mt_source.len() < 3 {
@@ -554,8 +517,8 @@ impl FrameDecoder {
/// Read bytes from the decode_buffer that are no longer needed. While the frame is not yet finished
/// this will retain window_size bytes, else it will drain it completely
-impl std::io::Read for FrameDecoder {
- fn read(&mut self, target: &mut [u8]) -> std::result::Result<usize, std::io::Error> {
+impl Read for FrameDecoder {
+ fn read(&mut self, target: &mut [u8]) -> Result<usize, Error> {
let state = match &mut self.state {
None => return Ok(0),
Some(s) => s,
diff --git a/vendor/ruzstd/src/fse/fse_decoder.rs b/vendor/ruzstd/src/fse/fse_decoder.rs
index 1847da728..21868ff84 100644
--- a/vendor/ruzstd/src/fse/fse_decoder.rs
+++ b/vendor/ruzstd/src/fse/fse_decoder.rs
@@ -1,7 +1,7 @@
use crate::decoding::bit_reader::BitReader;
use crate::decoding::bit_reader_reverse::{BitReaderReversed, GetBitsError};
+use alloc::vec::Vec;
-#[derive(Clone)]
pub struct FSETable {
pub decode: Vec<Entry>, //used to decode symbols, and calculate the next state
@@ -113,6 +113,15 @@ impl FSETable {
}
}
+ pub fn reinit_from(&mut self, other: &Self) {
+ self.reset();
+ self.symbol_counter.extend_from_slice(&other.symbol_counter);
+ self.symbol_probabilities
+ .extend_from_slice(&other.symbol_probabilities);
+ self.decode.extend_from_slice(&other.decode);
+ self.accuracy_log = other.accuracy_log;
+ }
+
pub fn reset(&mut self) {
self.symbol_counter.clear();
self.symbol_probabilities.clear();
diff --git a/vendor/ruzstd/src/huff0/huff0_decoder.rs b/vendor/ruzstd/src/huff0/huff0_decoder.rs
index 831ddd69c..a8aa74563 100644
--- a/vendor/ruzstd/src/huff0/huff0_decoder.rs
+++ b/vendor/ruzstd/src/huff0/huff0_decoder.rs
@@ -1,7 +1,7 @@
use crate::decoding::bit_reader_reverse::{BitReaderReversed, GetBitsError};
use crate::fse::{FSEDecoder, FSEDecoderError, FSETable, FSETableError};
+use alloc::vec::Vec;
-#[derive(Clone)]
pub struct HuffmanTable {
decode: Vec<Entry>,
@@ -134,6 +134,16 @@ impl HuffmanTable {
}
}
+ pub fn reinit_from(&mut self, other: &Self) {
+ self.reset();
+ self.decode.extend_from_slice(&other.decode);
+ self.weights.extend_from_slice(&other.weights);
+ self.max_num_bits = other.max_num_bits;
+ self.bits.extend_from_slice(&other.bits);
+ self.rank_indexes.extend_from_slice(&other.rank_indexes);
+ self.fse_table.reinit_from(&other.fse_table);
+ }
+
pub fn reset(&mut self) {
self.decode.clear();
self.weights.clear();
@@ -182,12 +192,10 @@ impl HuffmanTable {
});
}
- if crate::VERBOSE {
- println!(
- "Building fse table for huffman weights used: {}",
- bytes_used_by_fse_header
- );
- }
+ vprintln!(
+ "Building fse table for huffman weights used: {}",
+ bytes_used_by_fse_header
+ );
let mut dec1 = FSEDecoder::new(&self.fse_table);
let mut dec2 = FSEDecoder::new(&self.fse_table);
diff --git a/vendor/ruzstd/src/io.rs b/vendor/ruzstd/src/io.rs
new file mode 100644
index 000000000..6970cd132
--- /dev/null
+++ b/vendor/ruzstd/src/io.rs
@@ -0,0 +1,2 @@
+#[cfg(feature = "std")]
+pub use std::io::{Error, ErrorKind, Read, Write};
diff --git a/vendor/ruzstd/src/io_nostd.rs b/vendor/ruzstd/src/io_nostd.rs
new file mode 100644
index 000000000..1e5d14166
--- /dev/null
+++ b/vendor/ruzstd/src/io_nostd.rs
@@ -0,0 +1,152 @@
+use alloc::boxed::Box;
+
+#[non_exhaustive]
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)]
+pub enum ErrorKind {
+ Interrupted,
+ UnexpectedEof,
+ WouldBlock,
+ Other,
+}
+
+impl ErrorKind {
+ fn as_str(&self) -> &'static str {
+ use ErrorKind::*;
+ match *self {
+ Interrupted => "operation interrupted",
+ UnexpectedEof => "unexpected end of file",
+ WouldBlock => "operation would block",
+ Other => "other error",
+ }
+ }
+}
+
+impl core::fmt::Display for ErrorKind {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ f.write_str(self.as_str())
+ }
+}
+
+#[derive(Debug)]
+pub struct Error {
+ kind: ErrorKind,
+ err: Option<Box<dyn core::error::Error + Send + Sync>>,
+}
+
+impl Error {
+ pub fn new<E>(kind: ErrorKind, err: E) -> Self
+ where
+ E: Into<Box<dyn core::error::Error + Send + Sync>>,
+ {
+ Self {
+ kind,
+ err: Some(err.into()),
+ }
+ }
+
+ pub fn from(kind: ErrorKind) -> Self {
+ Self { kind, err: None }
+ }
+
+ pub fn kind(&self) -> ErrorKind {
+ self.kind
+ }
+
+ pub fn get_ref(&self) -> Option<&(dyn core::error::Error + Send + Sync + 'static)> {
+ self.err.as_ref().map(|e| e.as_ref())
+ }
+
+ pub fn get_mut(&mut self) -> Option<&mut (dyn core::error::Error + Send + Sync + 'static)> {
+ self.err.as_mut().map(|e| e.as_mut())
+ }
+
+ pub fn into_inner(self) -> Option<Box<dyn core::error::Error + Send + Sync>> {
+ self.err
+ }
+}
+
+impl core::fmt::Display for Error {
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
+ f.write_str(self.kind.as_str())?;
+
+ if let Some(ref e) = self.err {
+ e.fmt(f)?;
+ }
+
+ Ok(())
+ }
+}
+
+impl core::error::Error for Error {}
+
+impl From<ErrorKind> for Error {
+ fn from(value: ErrorKind) -> Self {
+ Self::from(value)
+ }
+}
+
+pub trait Read {
+ fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error>;
+
+ fn read_exact(&mut self, mut buf: &mut [u8]) -> Result<(), Error> {
+ while !buf.is_empty() {
+ match self.read(buf) {
+ Ok(0) => break,
+ Ok(n) => {
+ let tmp = buf;
+ buf = &mut tmp[n..];
+ }
+ Err(ref e) if e.kind() == ErrorKind::Interrupted => {}
+ Err(e) => return Err(e),
+ }
+ }
+ if !buf.is_empty() {
+ Err(Error::from(ErrorKind::UnexpectedEof))
+ } else {
+ Ok(())
+ }
+ }
+}
+
+impl Read for &[u8] {
+ fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
+ let size = core::cmp::min(self.len(), buf.len());
+ let (to_copy, rest) = self.split_at(size);
+
+ if size == 1 {
+ buf[0] = to_copy[0];
+ } else {
+ buf[..size].copy_from_slice(to_copy);
+ }
+
+ *self = rest;
+ Ok(size)
+ }
+}
+
+impl<'a, T> Read for &'a mut T
+where
+ T: Read,
+{
+ fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
+ (*self).read(buf)
+ }
+}
+
+pub trait Write {
+ fn write(&mut self, buf: &[u8]) -> Result<usize, Error>;
+ fn flush(&mut self) -> Result<(), Error>;
+}
+
+impl<'a, T> Write for &'a mut T
+where
+ T: Write,
+{
+ fn write(&mut self, buf: &[u8]) -> Result<usize, Error> {
+ (*self).write(buf)
+ }
+
+ fn flush(&mut self) -> Result<(), Error> {
+ (*self).flush()
+ }
+}
diff --git a/vendor/ruzstd/src/lib.rs b/vendor/ruzstd/src/lib.rs
index e079ad31b..aecef623b 100644
--- a/vendor/ruzstd/src/lib.rs
+++ b/vendor/ruzstd/src/lib.rs
@@ -1,4 +1,23 @@
+#![no_std]
#![deny(trivial_casts, trivial_numeric_casts, rust_2018_idioms)]
+#![cfg_attr(not(feature = "std"), feature(error_in_core))]
+
+#[cfg(feature = "std")]
+extern crate std;
+
+extern crate alloc;
+
+#[cfg(feature = "std")]
+pub const VERBOSE: bool = false;
+
+macro_rules! vprintln {
+ ($($x:expr),*) => {
+ #[cfg(feature = "std")]
+ if crate::VERBOSE {
+ std::println!($($x),*);
+ }
+ }
+}
pub mod blocks;
pub mod decoding;
@@ -9,7 +28,15 @@ pub mod huff0;
pub mod streaming_decoder;
mod tests;
-pub const VERBOSE: bool = false;
+#[cfg(feature = "std")]
+pub mod io;
+
+#[cfg(not(feature = "std"))]
+pub mod io_nostd;
+
+#[cfg(not(feature = "std"))]
+pub use io_nostd as io;
+
pub use frame_decoder::BlockDecodingStrategy;
pub use frame_decoder::FrameDecoder;
pub use streaming_decoder::StreamingDecoder;
diff --git a/vendor/ruzstd/src/streaming_decoder.rs b/vendor/ruzstd/src/streaming_decoder.rs
index 2613a363b..f345a897c 100644
--- a/vendor/ruzstd/src/streaming_decoder.rs
+++ b/vendor/ruzstd/src/streaming_decoder.rs
@@ -1,27 +1,33 @@
+use core::borrow::BorrowMut;
+
use crate::frame_decoder::{BlockDecodingStrategy, FrameDecoder, FrameDecoderError};
-use std::io::Read;
+use crate::io::{Error, ErrorKind, Read};
/// High level decoder that implements a io::Read that can be used with
/// io::Read::read_to_end / io::Read::read_exact or passing this to another library / module as a source for the decoded content
///
/// The lower level FrameDecoder by comparison allows for finer grained control but need sto have it's decode_blocks method called continously
/// to decode the zstd-frame.
-pub struct StreamingDecoder<READ: Read> {
- pub decoder: FrameDecoder,
+pub struct StreamingDecoder<READ: Read, DEC: BorrowMut<FrameDecoder>> {
+ pub decoder: DEC,
source: READ,
}
-impl<READ: Read> StreamingDecoder<READ> {
- pub fn new(mut source: READ) -> Result<StreamingDecoder<READ>, FrameDecoderError> {
- let mut decoder = FrameDecoder::new();
- decoder.init(&mut source)?;
+impl<READ: Read, DEC: BorrowMut<FrameDecoder>> StreamingDecoder<READ, DEC> {
+ pub fn new_with_decoder(
+ mut source: READ,
+ mut decoder: DEC,
+ ) -> Result<StreamingDecoder<READ, DEC>, FrameDecoderError> {
+ decoder.borrow_mut().init(&mut source)?;
Ok(StreamingDecoder { decoder, source })
}
+}
- pub fn new_with_decoder(
+impl<READ: Read> StreamingDecoder<READ, FrameDecoder> {
+ pub fn new(
mut source: READ,
- mut decoder: FrameDecoder,
- ) -> Result<StreamingDecoder<READ>, FrameDecoderError> {
+ ) -> Result<StreamingDecoder<READ, FrameDecoder>, FrameDecoderError> {
+ let mut decoder = FrameDecoder::new();
decoder.init(&mut source)?;
Ok(StreamingDecoder { decoder, source })
}
@@ -31,9 +37,10 @@ impl<READ: Read> StreamingDecoder<READ> {
}
}
-impl<READ: Read> Read for StreamingDecoder<READ> {
- fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
- if self.decoder.is_finished() && self.decoder.can_collect() == 0 {
+impl<READ: Read, DEC: BorrowMut<FrameDecoder>> Read for StreamingDecoder<READ, DEC> {
+ fn read(&mut self, buf: &mut [u8]) -> Result<usize, Error> {
+ let decoder = self.decoder.borrow_mut();
+ if decoder.is_finished() && decoder.can_collect() == 0 {
//No more bytes can ever be decoded
return Ok(0);
}
@@ -43,24 +50,24 @@ impl<READ: Read> Read for StreamingDecoder<READ> {
// So we need to call this until we can actually collect enough bytes
// TODO add BlockDecodingStrategy::UntilCollectable(usize) that pushes this logic into the decode_blocks function
- while self.decoder.can_collect() < buf.len() && !self.decoder.is_finished() {
+ while decoder.can_collect() < buf.len() && !decoder.is_finished() {
//More bytes can be decoded
- let additional_bytes_needed = buf.len() - self.decoder.can_collect();
- match self.decoder.decode_blocks(
+ let additional_bytes_needed = buf.len() - decoder.can_collect();
+ match decoder.decode_blocks(
&mut self.source,
BlockDecodingStrategy::UptoBytes(additional_bytes_needed),
) {
Ok(_) => { /*Nothing to do*/ }
Err(e) => {
- let err = std::io::Error::new(
- std::io::ErrorKind::Other,
- format!("Error in the zstd decoder: {:?}", e),
+ let err = Error::new(
+ ErrorKind::Other,
+ alloc::format!("Error in the zstd decoder: {:?}", e),
);
return Err(err);
}
}
}
- self.decoder.read(buf)
+ decoder.read(buf)
}
}
diff --git a/vendor/ruzstd/src/tests/decode_corpus.rs b/vendor/ruzstd/src/tests/decode_corpus.rs
index ea1a723a3..a8e21683b 100644
--- a/vendor/ruzstd/src/tests/decode_corpus.rs
+++ b/vendor/ruzstd/src/tests/decode_corpus.rs
@@ -1,8 +1,13 @@
#[test]
fn test_decode_corpus_files() {
+ extern crate std;
use crate::frame_decoder;
+ use alloc::borrow::ToOwned;
+ use alloc::string::{String, ToString};
+ use alloc::vec::Vec;
use std::fs;
use std::io::Read;
+ use std::println;
let mut success_counter = 0;
let mut fail_counter_diff = 0;
diff --git a/vendor/ruzstd/src/tests/dict_test.rs b/vendor/ruzstd/src/tests/dict_test.rs
index a8a5fd46b..5bf76f91e 100644
--- a/vendor/ruzstd/src/tests/dict_test.rs
+++ b/vendor/ruzstd/src/tests/dict_test.rs
@@ -1,6 +1,7 @@
#[test]
fn test_dict_parsing() {
use crate::decoding::dictionary::Dictionary;
+ use alloc::vec;
let mut raw = vec![0u8; 8];
// correct magic num
@@ -75,9 +76,14 @@ fn test_dict_parsing() {
#[test]
fn test_dict_decoding() {
+ extern crate std;
use crate::frame_decoder;
+ use alloc::borrow::ToOwned;
+ use alloc::string::{String, ToString};
+ use alloc::vec::Vec;
use std::fs;
use std::io::Read;
+ use std::println;
let mut success_counter = 0;
let mut fail_counter_diff = 0;
@@ -99,7 +105,8 @@ fn test_dict_decoding() {
});
let mut frame_dec = frame_decoder::FrameDecoder::new();
- frame_dec.add_dict(&dict).unwrap();
+ let dict = crate::decoding::dictionary::Dictionary::decode_dict(&dict).unwrap();
+ frame_dec.add_dict(dict).unwrap();
for file in files {
let f = file.unwrap();
diff --git a/vendor/ruzstd/src/tests/fuzz_regressions.rs b/vendor/ruzstd/src/tests/fuzz_regressions.rs
index 2e293af4d..bc675cad3 100644
--- a/vendor/ruzstd/src/tests/fuzz_regressions.rs
+++ b/vendor/ruzstd/src/tests/fuzz_regressions.rs
@@ -1,6 +1,8 @@
#[test]
fn test_all_artifacts() {
+ extern crate std;
use crate::frame_decoder;
+ use std::borrow::ToOwned;
use std::fs;
use std::fs::File;
diff --git a/vendor/ruzstd/src/tests/mod.rs b/vendor/ruzstd/src/tests/mod.rs
index 070565c36..0f25073e5 100644
--- a/vendor/ruzstd/src/tests/mod.rs
+++ b/vendor/ruzstd/src/tests/mod.rs
@@ -1,4 +1,28 @@
#[cfg(test)]
+use alloc::vec;
+
+#[cfg(test)]
+use alloc::vec::Vec;
+
+#[cfg(test)]
+extern crate std;
+
+#[cfg(all(test, not(feature = "std")))]
+impl crate::io_nostd::Read for std::fs::File {
+ fn read(&mut self, buf: &mut [u8]) -> Result<usize, crate::io_nostd::Error> {
+ std::io::Read::read(self, buf).map_err(|e| {
+ if e.get_ref().is_none() {
+ crate::io_nostd::Error::from(crate::io_nostd::ErrorKind::Other)
+ } else {
+ crate::io_nostd::Error::new(
+ crate::io_nostd::ErrorKind::Other,
+ e.into_inner().unwrap(),
+ )
+ }
+ })
+ }
+}
+
#[test]
fn skippable_frame() {
use crate::frame;
@@ -34,8 +58,7 @@ fn test_frame_header_reading() {
use std::fs;
let mut content = fs::File::open("./decodecorpus_files/z000088.zst").unwrap();
- let (frame, _) = frame::read_frame_header(&mut content).unwrap();
- frame.check_valid().unwrap();
+ let (_frame, _) = frame::read_frame_header(&mut content).unwrap();
}
#[test]
@@ -45,8 +68,7 @@ fn test_block_header_reading() {
use std::fs;
let mut content = fs::File::open("./decodecorpus_files/z000088.zst").unwrap();
- let (frame, _) = frame::read_frame_header(&mut content).unwrap();
- frame.check_valid().unwrap();
+ let (_frame, _) = frame::read_frame_header(&mut content).unwrap();
let mut block_dec = decoding::block_decoder::new();
let block_header = block_dec.read_block_header(&mut content).unwrap();
@@ -131,16 +153,16 @@ fn test_decode_from_to() {
match frame_dec.get_checksum_from_data() {
Some(chksum) => {
if frame_dec.get_calculated_checksum().unwrap() != chksum {
- println!(
+ std::println!(
"Checksum did not match! From data: {}, calculated while decoding: {}\n",
chksum,
frame_dec.get_calculated_checksum().unwrap()
);
} else {
- println!("Checksums are ok!\n");
+ std::println!("Checksums are ok!\n");
}
}
- None => println!("No checksums to test\n"),
+ None => std::println!("No checksums to test\n"),
}
let original_f = File::open("./decodecorpus_files/z000088").unwrap();
@@ -163,7 +185,7 @@ fn test_decode_from_to() {
for idx in 0..min {
if original[idx] != result[idx] {
counter += 1;
- //println!(
+ //std::println!(
// "Original {:3} not equal to result {:3} at byte: {}",
// original[idx], result[idx], idx,
//);
@@ -204,10 +226,10 @@ fn test_specific_file() {
let original_f = fs::File::open("./decodecorpus_files/z000088").unwrap();
let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
- println!("Results for file: {}", path);
+ std::println!("Results for file: {}", path);
if original.len() != result.len() {
- println!(
+ std::println!(
"Result has wrong length: {}, should be: {}",
result.len(),
original.len()
@@ -223,18 +245,19 @@ fn test_specific_file() {
for idx in 0..min {
if original[idx] != result[idx] {
counter += 1;
- //println!(
+ //std::println!(
// "Original {:3} not equal to result {:3} at byte: {}",
// original[idx], result[idx], idx,
//);
}
}
if counter > 0 {
- println!("Result differs in at least {} bytes from original", counter);
+ std::println!("Result differs in at least {} bytes from original", counter);
}
}
#[test]
+#[cfg(feature = "std")]
fn test_streaming() {
use std::fs;
use std::io::Read;
@@ -265,7 +288,7 @@ fn test_streaming() {
for idx in 0..min {
if original[idx] != result[idx] {
counter += 1;
- //println!(
+ //std::println!(
// "Original {:3} not equal to result {:3} at byte: {}",
// original[idx], result[idx], idx,
//);
@@ -288,7 +311,91 @@ fn test_streaming() {
let original_f = fs::File::open("./decodecorpus_files/z000068").unwrap();
let original: Vec<u8> = original_f.bytes().map(|x| x.unwrap()).collect();
- println!("Results for file:");
+ std::println!("Results for file:");
+
+ if original.len() != result.len() {
+ panic!(
+ "Result has wrong length: {}, should be: {}",
+ result.len(),
+ original.len()
+ );
+ }
+
+ let mut counter = 0;
+ let min = if original.len() < result.len() {
+ original.len()
+ } else {
+ result.len()
+ };
+ for idx in 0..min {
+ if original[idx] != result[idx] {
+ counter += 1;
+ //std::println!(
+ // "Original {:3} not equal to result {:3} at byte: {}",
+ // original[idx], result[idx], idx,
+ //);
+ }
+ }
+ if counter > 0 {
+ panic!("Result differs in at least {} bytes from original", counter);
+ }
+}
+
+#[test]
+#[cfg(not(feature = "std"))]
+fn test_streaming_no_std() {
+ use crate::io::Read;
+
+ let content = include_bytes!("../../decodecorpus_files/z000088.zst");
+ let mut content = content.as_slice();
+ let mut stream = crate::streaming_decoder::StreamingDecoder::new(&mut content).unwrap();
+
+ let original = include_bytes!("../../decodecorpus_files/z000088");
+ let mut result = Vec::new();
+ result.resize(original.len(), 0);
+ Read::read_exact(&mut stream, &mut result).unwrap();
+
+ if original.len() != result.len() {
+ panic!(
+ "Result has wrong length: {}, should be: {}",
+ result.len(),
+ original.len()
+ );
+ }
+
+ let mut counter = 0;
+ let min = if original.len() < result.len() {
+ original.len()
+ } else {
+ result.len()
+ };
+ for idx in 0..min {
+ if original[idx] != result[idx] {
+ counter += 1;
+ //std::println!(
+ // "Original {:3} not equal to result {:3} at byte: {}",
+ // original[idx], result[idx], idx,
+ //);
+ }
+ }
+ if counter > 0 {
+ panic!("Result differs in at least {} bytes from original", counter);
+ }
+
+ // Test resetting to a new file while keeping the old decoder
+
+ let content = include_bytes!("../../decodecorpus_files/z000068.zst");
+ let mut content = content.as_slice();
+ let mut stream =
+ crate::streaming_decoder::StreamingDecoder::new_with_decoder(&mut content, stream.inner())
+ .unwrap();
+
+ let original = include_bytes!("../../decodecorpus_files/z000068");
+ let mut result = Vec::new();
+ result.resize(original.len(), 0);
+ Read::read_exact(&mut stream, &mut result).unwrap();
+
+ std::println!("Results for file:");
if original.len() != result.len() {
panic!(
@@ -307,7 +414,7 @@ fn test_streaming() {
for idx in 0..min {
if original[idx] != result[idx] {
counter += 1;
- //println!(
+ //std::println!(
// "Original {:3} not equal to result {:3} at byte: {}",
// original[idx], result[idx], idx,
//);