From d1b2d29528b7794b41e66fc2136e395a02f8529b Mon Sep 17 00:00:00 2001 From: Daniel Baumann Date: Thu, 30 May 2024 05:59:35 +0200 Subject: Merging upstream version 1.73.0+dfsg1. Signed-off-by: Daniel Baumann --- vendor/ruzstd/src/blocks/block.rs | 4 +- vendor/ruzstd/src/blocks/literals_section.rs | 4 +- vendor/ruzstd/src/blocks/sequence_section.rs | 4 +- vendor/ruzstd/src/decoding/block_decoder.rs | 42 ++--- vendor/ruzstd/src/decoding/decodebuffer.rs | 41 ++--- vendor/ruzstd/src/decoding/dictionary.rs | 3 +- .../src/decoding/literals_section_decoder.rs | 5 +- vendor/ruzstd/src/decoding/ringbuffer.rs | 19 +-- vendor/ruzstd/src/decoding/scratch.rs | 38 ++--- vendor/ruzstd/src/decoding/sequence_execution.rs | 2 - .../src/decoding/sequence_section_decoder.rs | 68 +++----- vendor/ruzstd/src/frame.rs | 160 ++++++------------- vendor/ruzstd/src/frame_decoder.rs | 173 ++++++++------------- vendor/ruzstd/src/fse/fse_decoder.rs | 11 +- vendor/ruzstd/src/huff0/huff0_decoder.rs | 22 ++- vendor/ruzstd/src/io.rs | 2 + vendor/ruzstd/src/io_nostd.rs | 152 ++++++++++++++++++ vendor/ruzstd/src/lib.rs | 29 +++- vendor/ruzstd/src/streaming_decoder.rs | 47 +++--- vendor/ruzstd/src/tests/decode_corpus.rs | 5 + vendor/ruzstd/src/tests/dict_test.rs | 9 +- vendor/ruzstd/src/tests/fuzz_regressions.rs | 2 + vendor/ruzstd/src/tests/mod.rs | 137 ++++++++++++++-- 23 files changed, 590 insertions(+), 389 deletions(-) create mode 100644 vendor/ruzstd/src/io.rs create mode 100644 vendor/ruzstd/src/io_nostd.rs (limited to 'vendor/ruzstd/src') diff --git a/vendor/ruzstd/src/blocks/block.rs b/vendor/ruzstd/src/blocks/block.rs index 9c872ebf9..078eb44ee 100644 --- a/vendor/ruzstd/src/blocks/block.rs +++ b/vendor/ruzstd/src/blocks/block.rs @@ -6,8 +6,8 @@ pub enum BlockType { Reserved, } -impl std::fmt::Display for BlockType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { +impl core::fmt::Display for BlockType { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { match self { BlockType::Compressed => write!(f, "Compressed"), BlockType::Raw => write!(f, "Raw"), diff --git a/vendor/ruzstd/src/blocks/literals_section.rs b/vendor/ruzstd/src/blocks/literals_section.rs index 9d8307223..159e3b303 100644 --- a/vendor/ruzstd/src/blocks/literals_section.rs +++ b/vendor/ruzstd/src/blocks/literals_section.rs @@ -25,8 +25,8 @@ pub enum LiteralsSectionParseError { NotEnoughBytes { have: usize, need: u8 }, } -impl std::fmt::Display for LiteralsSectionType { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { +impl core::fmt::Display for LiteralsSectionType { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { match self { LiteralsSectionType::Compressed => write!(f, "Compressed"), LiteralsSectionType::Raw => write!(f, "Raw"), diff --git a/vendor/ruzstd/src/blocks/sequence_section.rs b/vendor/ruzstd/src/blocks/sequence_section.rs index 28b9efff2..544755e03 100644 --- a/vendor/ruzstd/src/blocks/sequence_section.rs +++ b/vendor/ruzstd/src/blocks/sequence_section.rs @@ -10,8 +10,8 @@ pub struct Sequence { pub of: u32, } -impl std::fmt::Display for Sequence { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> Result<(), std::fmt::Error> { +impl core::fmt::Display for Sequence { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> Result<(), core::fmt::Error> { write!(f, "LL: {}, ML: {}, OF: {}", self.ll, self.ml, self.of) } } diff --git a/vendor/ruzstd/src/decoding/block_decoder.rs b/vendor/ruzstd/src/decoding/block_decoder.rs index 11a4c28c1..0fbce8f48 100644 --- a/vendor/ruzstd/src/decoding/block_decoder.rs +++ b/vendor/ruzstd/src/decoding/block_decoder.rs @@ -11,7 +11,7 @@ use crate::blocks::literals_section::LiteralsSectionParseError; use crate::blocks::sequence_section::SequencesHeaderParseError; use crate::decoding::scratch::DecoderScratch; use crate::decoding::sequence_execution::execute_sequences; -use std::io::{self, Read}; +use crate::io::{self, Read}; pub struct BlockDecoder { header_buffer: [u8; 3], @@ -203,12 +203,12 @@ impl BlockDecoder { let mut section = LiteralsSection::new(); let bytes_in_literals_header = section.parse_from_header(raw)?; let raw = &raw[bytes_in_literals_header as usize..]; - if crate::VERBOSE { - println!( - "Found {} literalssection with regenerated size: {}, and compressed size: {:?}", - section.ls_type, section.regenerated_size, section.compressed_size - ); - } + vprintln!( + "Found {} literalssection with regenerated size: {}, and compressed size: {:?}", + section.ls_type, + section.regenerated_size, + section.compressed_size + ); let upper_limit_for_literals = match section.compressed_size { Some(x) => x as usize, @@ -227,9 +227,7 @@ impl BlockDecoder { } let raw_literals = &raw[..upper_limit_for_literals]; - if crate::VERBOSE { - println!("Slice for literals: {}", raw_literals.len()); - } + vprintln!("Slice for literals: {}", raw_literals.len()); workspace.literals_buffer.clear(); //all literals of the previous block must have been used in the sequence execution anyways. just be defensive here let bytes_used_in_literals_section = decode_literals( @@ -247,20 +245,16 @@ impl BlockDecoder { assert!(bytes_used_in_literals_section == upper_limit_for_literals as u32); let raw = &raw[upper_limit_for_literals..]; - if crate::VERBOSE { - println!("Slice for sequences with headers: {}", raw.len()); - } + vprintln!("Slice for sequences with headers: {}", raw.len()); let mut seq_section = SequencesHeader::new(); let bytes_in_sequence_header = seq_section.parse_from_header(raw)?; let raw = &raw[bytes_in_sequence_header as usize..]; - if crate::VERBOSE { - println!( - "Found sequencessection with sequences: {} and size: {}", - seq_section.num_sequences, - raw.len() - ); - } + vprintln!( + "Found sequencessection with sequences: {} and size: {}", + seq_section.num_sequences, + raw.len() + ); assert!( u32::from(bytes_in_literals_header) @@ -269,9 +263,7 @@ impl BlockDecoder { + raw.len() as u32 == header.content_size ); - if crate::VERBOSE { - println!("Slice for sequences: {}", raw.len()); - } + vprintln!("Slice for sequences: {}", raw.len()); if seq_section.num_sequences != 0 { decode_sequences( @@ -280,9 +272,7 @@ impl BlockDecoder { &mut workspace.fse, &mut workspace.sequences, )?; - if crate::VERBOSE { - println!("Executing sequences"); - } + vprintln!("Executing sequences"); execute_sequences(workspace)?; } else { workspace.buffer.push(&workspace.literals_buffer); diff --git a/vendor/ruzstd/src/decoding/decodebuffer.rs b/vendor/ruzstd/src/decoding/decodebuffer.rs index 0dea7015e..33cc58c30 100644 --- a/vendor/ruzstd/src/decoding/decodebuffer.rs +++ b/vendor/ruzstd/src/decoding/decodebuffer.rs @@ -1,5 +1,6 @@ -use std::hash::Hasher; -use std::io; +use crate::io::{Error, Read, Write}; +use alloc::vec::Vec; +use core::hash::Hasher; use twox_hash::XxHash64; @@ -23,8 +24,8 @@ pub enum DecodebufferError { OffsetTooBig { offset: usize, buf_len: usize }, } -impl io::Read for Decodebuffer { - fn read(&mut self, target: &mut [u8]) -> io::Result { +impl Read for Decodebuffer { + fn read(&mut self, target: &mut [u8]) -> Result { let max_amount = self.can_drain_to_window_size().unwrap_or(0); let amount = max_amount.min(target.len()); @@ -176,7 +177,7 @@ impl Decodebuffer { } } - pub fn drain_to_window_size_writer(&mut self, mut sink: impl io::Write) -> io::Result { + pub fn drain_to_window_size_writer(&mut self, mut sink: impl Write) -> Result { match self.can_drain_to_window_size() { None => Ok(0), Some(can_drain) => { @@ -199,14 +200,14 @@ impl Decodebuffer { vec } - pub fn drain_to_writer(&mut self, mut sink: impl io::Write) -> io::Result { + pub fn drain_to_writer(&mut self, mut sink: impl Write) -> Result { let len = self.buffer.len(); self.drain_to(len, |buf| write_all_bytes(&mut sink, buf))?; Ok(len) } - pub fn read_all(&mut self, target: &mut [u8]) -> io::Result { + pub fn read_all(&mut self, target: &mut [u8]) -> Result { let amount = self.buffer.len().min(target.len()); let mut written = 0; @@ -224,8 +225,8 @@ impl Decodebuffer { fn drain_to( &mut self, amount: usize, - mut write_bytes: impl FnMut(&[u8]) -> (usize, io::Result<()>), - ) -> io::Result<()> { + mut write_bytes: impl FnMut(&[u8]) -> (usize, Result<(), Error>), + ) -> Result<(), Error> { if amount == 0 { return Ok(()); } @@ -280,7 +281,7 @@ impl Decodebuffer { } /// Like Write::write_all but returns partial write length even on error -fn write_all_bytes(mut sink: impl io::Write, buf: &[u8]) -> (usize, io::Result<()>) { +fn write_all_bytes(mut sink: impl Write, buf: &[u8]) -> (usize, Result<(), Error>) { let mut written = 0; while written < buf.len() { match sink.write(&buf[written..]) { @@ -294,7 +295,11 @@ fn write_all_bytes(mut sink: impl io::Write, buf: &[u8]) -> (usize, io::Result<( #[cfg(test)] mod tests { use super::Decodebuffer; - use std::io::Write; + use crate::io::{Error, ErrorKind, Write}; + + extern crate std; + use alloc::vec; + use alloc::vec::Vec; #[test] fn short_writer() { @@ -304,7 +309,7 @@ mod tests { } impl Write for ShortWriter { - fn write(&mut self, buf: &[u8]) -> std::result::Result { + fn write(&mut self, buf: &[u8]) -> std::result::Result { if buf.len() > self.write_len { self.buf.extend_from_slice(&buf[..self.write_len]); Ok(self.write_len) @@ -314,7 +319,7 @@ mod tests { } } - fn flush(&mut self) -> std::result::Result<(), std::io::Error> { + fn flush(&mut self) -> std::result::Result<(), Error> { Ok(()) } } @@ -352,18 +357,18 @@ mod tests { } impl Write for WouldblockWriter { - fn write(&mut self, buf: &[u8]) -> std::result::Result { + fn write(&mut self, buf: &[u8]) -> std::result::Result { if self.last_blocked < self.block_every { self.buf.extend_from_slice(buf); self.last_blocked += 1; Ok(buf.len()) } else { self.last_blocked = 0; - Err(std::io::Error::from(std::io::ErrorKind::WouldBlock)) + Err(Error::from(ErrorKind::WouldBlock)) } } - fn flush(&mut self) -> std::result::Result<(), std::io::Error> { + fn flush(&mut self) -> std::result::Result<(), Error> { Ok(()) } } @@ -390,7 +395,7 @@ mod tests { } } Err(e) => { - if e.kind() == std::io::ErrorKind::WouldBlock { + if e.kind() == ErrorKind::WouldBlock { continue; } else { panic!("Unexpected error {:?}", e); @@ -410,7 +415,7 @@ mod tests { } } Err(e) => { - if e.kind() == std::io::ErrorKind::WouldBlock { + if e.kind() == ErrorKind::WouldBlock { continue; } else { panic!("Unexpected error {:?}", e); diff --git a/vendor/ruzstd/src/decoding/dictionary.rs b/vendor/ruzstd/src/decoding/dictionary.rs index 51fbcdf0b..aa6769371 100644 --- a/vendor/ruzstd/src/decoding/dictionary.rs +++ b/vendor/ruzstd/src/decoding/dictionary.rs @@ -1,4 +1,5 @@ -use std::convert::TryInto; +use alloc::vec::Vec; +use core::convert::TryInto; use crate::decoding::scratch::FSEScratch; use crate::decoding::scratch::HuffmanScratch; diff --git a/vendor/ruzstd/src/decoding/literals_section_decoder.rs b/vendor/ruzstd/src/decoding/literals_section_decoder.rs index d947f87eb..bd7fb18ea 100644 --- a/vendor/ruzstd/src/decoding/literals_section_decoder.rs +++ b/vendor/ruzstd/src/decoding/literals_section_decoder.rs @@ -2,6 +2,7 @@ use super::super::blocks::literals_section::{LiteralsSection, LiteralsSectionTyp use super::bit_reader_reverse::{BitReaderReversed, GetBitsError}; use super::scratch::HuffmanScratch; use crate::huff0::{HuffmanDecoder, HuffmanDecoderError, HuffmanTableError}; +use alloc::vec::Vec; #[derive(Debug, thiserror::Error)] #[non_exhaustive] @@ -75,9 +76,7 @@ fn decompress_literals( LiteralsSectionType::Compressed => { //read Huffman tree description bytes_read += scratch.table.build_decoder(source)?; - if crate::VERBOSE { - println!("Built huffman table using {} bytes", bytes_read); - } + vprintln!("Built huffman table using {} bytes", bytes_read); } LiteralsSectionType::Treeless => { if scratch.table.max_num_bits == 0 { diff --git a/vendor/ruzstd/src/decoding/ringbuffer.rs b/vendor/ruzstd/src/decoding/ringbuffer.rs index 9e3e9ba5e..fc9a2e330 100644 --- a/vendor/ruzstd/src/decoding/ringbuffer.rs +++ b/vendor/ruzstd/src/decoding/ringbuffer.rs @@ -1,4 +1,5 @@ -use std::{alloc::Layout, ptr::NonNull, slice}; +use alloc::alloc::{alloc, dealloc}; +use core::{alloc::Layout, ptr::NonNull, slice}; pub struct RingBuffer { buf: NonNull, @@ -70,7 +71,7 @@ impl RingBuffer { // alloc the new memory region and panic if alloc fails // TODO maybe rework this to generate an error? let new_buf = unsafe { - let new_buf = std::alloc::alloc(new_layout); + let new_buf = alloc(new_layout); NonNull::new(new_buf).expect("Allocating new space for the ringbuffer failed") }; @@ -85,7 +86,7 @@ impl RingBuffer { .as_ptr() .add(s1_len) .copy_from_nonoverlapping(s2_ptr, s2_len); - std::alloc::dealloc(self.buf.as_ptr(), current_layout); + dealloc(self.buf.as_ptr(), current_layout); } self.tail = s1_len + s2_len; @@ -341,7 +342,7 @@ impl Drop for RingBuffer { let current_layout = unsafe { Layout::array::(self.cap).unwrap_unchecked() }; unsafe { - std::alloc::dealloc(self.buf.as_ptr(), current_layout); + dealloc(self.buf.as_ptr(), current_layout); } } } @@ -448,8 +449,8 @@ unsafe fn copy_with_nobranch_check( f1_ptr.copy_from_nonoverlapping(m1_ptr, m1_in_f1); f2_ptr.copy_from_nonoverlapping(m1_ptr.add(m1_in_f1), m1_in_f2); } - 6 => std::hint::unreachable_unchecked(), - 7 => std::hint::unreachable_unchecked(), + 6 => core::hint::unreachable_unchecked(), + 7 => core::hint::unreachable_unchecked(), 9 => { f1_ptr.copy_from_nonoverlapping(m1_ptr, m1_in_f1); f2_ptr.copy_from_nonoverlapping(m2_ptr, m2_in_f2); @@ -480,9 +481,9 @@ unsafe fn copy_with_nobranch_check( .add(m1_in_f2) .copy_from_nonoverlapping(m2_ptr, m2_in_f2); } - 14 => std::hint::unreachable_unchecked(), - 15 => std::hint::unreachable_unchecked(), - _ => std::hint::unreachable_unchecked(), + 14 => core::hint::unreachable_unchecked(), + 15 => core::hint::unreachable_unchecked(), + _ => core::hint::unreachable_unchecked(), } } diff --git a/vendor/ruzstd/src/decoding/scratch.rs b/vendor/ruzstd/src/decoding/scratch.rs index 2bd753bde..35d5c61ef 100644 --- a/vendor/ruzstd/src/decoding/scratch.rs +++ b/vendor/ruzstd/src/decoding/scratch.rs @@ -3,6 +3,7 @@ use super::decodebuffer::Decodebuffer; use crate::decoding::dictionary::Dictionary; use crate::fse::FSETable; use crate::huff0::HuffmanTable; +use alloc::vec::Vec; pub struct DecoderScratch { pub huf: HuffmanScratch, @@ -56,30 +57,17 @@ impl DecoderScratch { self.huf.table.reset(); } - pub fn use_dict(&mut self, dict: &Dictionary) { - self.fse = dict.fse.clone(); - self.huf = dict.huf.clone(); + pub fn init_from_dict(&mut self, dict: &Dictionary) { + self.fse.reinit_from(&dict.fse); + self.huf.table.reinit_from(&dict.huf.table); self.offset_hist = dict.offset_hist; - self.buffer.dict_content = dict.dict_content.clone(); - } - - /// parses the dictionary and set the tables - /// it returns the dict_id for checking with the frame's dict_id - pub fn load_dict( - &mut self, - raw: &[u8], - ) -> Result { - let dict = super::dictionary::Dictionary::decode_dict(raw)?; - - self.huf = dict.huf.clone(); - self.fse = dict.fse.clone(); - self.offset_hist = dict.offset_hist; - self.buffer.dict_content = dict.dict_content.clone(); - Ok(dict.id) + self.buffer.dict_content.clear(); + self.buffer + .dict_content + .extend_from_slice(&dict.dict_content); } } -#[derive(Clone)] pub struct HuffmanScratch { pub table: HuffmanTable, } @@ -98,7 +86,6 @@ impl Default for HuffmanScratch { } } -#[derive(Clone)] pub struct FSEScratch { pub offsets: FSETable, pub of_rle: Option, @@ -119,6 +106,15 @@ impl FSEScratch { ml_rle: None, } } + + pub fn reinit_from(&mut self, other: &Self) { + self.offsets.reinit_from(&other.offsets); + self.literal_lengths.reinit_from(&other.literal_lengths); + self.match_lengths.reinit_from(&other.match_lengths); + self.of_rle = other.of_rle; + self.ll_rle = other.ll_rle; + self.ml_rle = other.ml_rle; + } } impl Default for FSEScratch { diff --git a/vendor/ruzstd/src/decoding/sequence_execution.rs b/vendor/ruzstd/src/decoding/sequence_execution.rs index 19247ec62..5946df18c 100644 --- a/vendor/ruzstd/src/decoding/sequence_execution.rs +++ b/vendor/ruzstd/src/decoding/sequence_execution.rs @@ -18,8 +18,6 @@ pub fn execute_sequences(scratch: &mut DecoderScratch) -> Result<(), ExecuteSequ for idx in 0..scratch.sequences.len() { let seq = scratch.sequences[idx]; - if crate::VERBOSE {} - //println!("{}: {}", idx, seq); if seq.ll > 0 { let high = literals_copy_counter + seq.ll as usize; diff --git a/vendor/ruzstd/src/decoding/sequence_section_decoder.rs b/vendor/ruzstd/src/decoding/sequence_section_decoder.rs index 3d5990c05..6c366fba1 100644 --- a/vendor/ruzstd/src/decoding/sequence_section_decoder.rs +++ b/vendor/ruzstd/src/decoding/sequence_section_decoder.rs @@ -4,6 +4,7 @@ use super::super::blocks::sequence_section::SequencesHeader; use super::bit_reader_reverse::{BitReaderReversed, GetBitsError}; use super::scratch::FSEScratch; use crate::fse::{FSEDecoder, FSEDecoderError, FSETableError}; +use alloc::vec::Vec; #[derive(Debug, thiserror::Error)] #[non_exhaustive] @@ -42,9 +43,7 @@ pub fn decode_sequences( ) -> Result<(), DecodeSequenceError> { let bytes_read = maybe_update_fse_tables(section, source, scratch)?; - if crate::VERBOSE { - println!("Updating tables used {} bytes", bytes_read); - } + vprintln!("Updating tables used {} bytes", bytes_read); let bit_stream = &source[bytes_read..]; @@ -319,16 +318,13 @@ fn maybe_update_fse_tables( ModeType::FSECompressed => { let bytes = scratch.literal_lengths.build_decoder(source, LL_MAX_LOG)?; bytes_read += bytes; - if crate::VERBOSE { - println!("Updating ll table"); - println!("Used bytes: {}", bytes); - } + + vprintln!("Updating ll table"); + vprintln!("Used bytes: {}", bytes); scratch.ll_rle = None; } ModeType::RLE => { - if crate::VERBOSE { - println!("Use RLE ll table"); - } + vprintln!("Use RLE ll table"); if source.is_empty() { return Err(DecodeSequenceError::MissingByteForRleLlTable); } @@ -336,9 +332,7 @@ fn maybe_update_fse_tables( scratch.ll_rle = Some(source[0]); } ModeType::Predefined => { - if crate::VERBOSE { - println!("Use predefined ll table"); - } + vprintln!("Use predefined ll table"); scratch.literal_lengths.build_from_probabilities( LL_DEFAULT_ACC_LOG, &Vec::from(&LITERALS_LENGTH_DEFAULT_DISTRIBUTION[..]), @@ -346,9 +340,7 @@ fn maybe_update_fse_tables( scratch.ll_rle = None; } ModeType::Repeat => { - if crate::VERBOSE { - println!("Repeat ll table"); - } + vprintln!("Repeat ll table"); /* Nothing to do */ } }; @@ -358,17 +350,13 @@ fn maybe_update_fse_tables( match modes.of_mode() { ModeType::FSECompressed => { let bytes = scratch.offsets.build_decoder(of_source, OF_MAX_LOG)?; - if crate::VERBOSE { - println!("Updating of table"); - println!("Used bytes: {}", bytes); - } + vprintln!("Updating of table"); + vprintln!("Used bytes: {}", bytes); bytes_read += bytes; scratch.of_rle = None; } ModeType::RLE => { - if crate::VERBOSE { - println!("Use RLE of table"); - } + vprintln!("Use RLE of table"); if of_source.is_empty() { return Err(DecodeSequenceError::MissingByteForRleOfTable); } @@ -376,9 +364,7 @@ fn maybe_update_fse_tables( scratch.of_rle = Some(of_source[0]); } ModeType::Predefined => { - if crate::VERBOSE { - println!("Use predefined of table"); - } + vprintln!("Use predefined of table"); scratch.offsets.build_from_probabilities( OF_DEFAULT_ACC_LOG, &Vec::from(&OFFSET_DEFAULT_DISTRIBUTION[..]), @@ -386,9 +372,7 @@ fn maybe_update_fse_tables( scratch.of_rle = None; } ModeType::Repeat => { - if crate::VERBOSE { - println!("Repeat of table"); - } + vprintln!("Repeat of table"); /* Nothing to do */ } }; @@ -399,16 +383,12 @@ fn maybe_update_fse_tables( ModeType::FSECompressed => { let bytes = scratch.match_lengths.build_decoder(ml_source, ML_MAX_LOG)?; bytes_read += bytes; - if crate::VERBOSE { - println!("Updating ml table"); - println!("Used bytes: {}", bytes); - } + vprintln!("Updating ml table"); + vprintln!("Used bytes: {}", bytes); scratch.ml_rle = None; } ModeType::RLE => { - if crate::VERBOSE { - println!("Use RLE ml table"); - } + vprintln!("Use RLE ml table"); if ml_source.is_empty() { return Err(DecodeSequenceError::MissingByteForRleMlTable); } @@ -416,9 +396,7 @@ fn maybe_update_fse_tables( scratch.ml_rle = Some(ml_source[0]); } ModeType::Predefined => { - if crate::VERBOSE { - println!("Use predefined ml table"); - } + vprintln!("Use predefined ml table"); scratch.match_lengths.build_from_probabilities( ML_DEFAULT_ACC_LOG, &Vec::from(&MATCH_LENGTH_DEFAULT_DISTRIBUTION[..]), @@ -426,9 +404,7 @@ fn maybe_update_fse_tables( scratch.ml_rle = None; } ModeType::Repeat => { - if crate::VERBOSE { - println!("Repeat ml table"); - } + vprintln!("Repeat ml table"); /* Nothing to do */ } }; @@ -463,10 +439,14 @@ fn test_ll_default() { ) .unwrap(); + #[cfg(feature = "std")] for idx in 0..table.decode.len() { - println!( + std::println!( "{:3}: {:3} {:3} {:3}", - idx, table.decode[idx].symbol, table.decode[idx].num_bits, table.decode[idx].base_line + idx, + table.decode[idx].symbol, + table.decode[idx].num_bits, + table.decode[idx].base_line ); } diff --git a/vendor/ruzstd/src/frame.rs b/vendor/ruzstd/src/frame.rs index 1eb3f8911..3810e4bca 100644 --- a/vendor/ruzstd/src/frame.rs +++ b/vendor/ruzstd/src/frame.rs @@ -1,20 +1,17 @@ -use std::convert::TryInto; -use std::io; - +use crate::io::{Error, Read}; pub const MAGIC_NUM: u32 = 0xFD2F_B528; pub const MIN_WINDOW_SIZE: u64 = 1024; pub const MAX_WINDOW_SIZE: u64 = (1 << 41) + 7 * (1 << 38); pub struct Frame { - magic_num: u32, pub header: FrameHeader, } pub struct FrameHeader { pub descriptor: FrameDescriptor, window_descriptor: u8, - dict_id: Vec, - frame_content_size: Vec, + dict_id: Option, + frame_content_size: u64, } pub struct FrameDescriptor(u8); @@ -97,7 +94,7 @@ pub enum FrameHeaderError { impl FrameHeader { pub fn window_size(&self) -> Result { if self.descriptor.single_segment_flag() { - self.frame_content_size() + Ok(self.frame_content_size()) } else { let exp = self.window_descriptor >> 3; let mantissa = self.window_descriptor & 0x7; @@ -120,93 +117,12 @@ impl FrameHeader { } } - pub fn dictionary_id(&self) -> Result, FrameHeaderError> { - if self.descriptor.dict_id_flag() == 0 { - Ok(None) - } else { - let bytes = self.descriptor.dictionary_id_bytes()?; - if self.dict_id.len() != bytes as usize { - Err(FrameHeaderError::DictIdTooSmall { - got: self.dict_id.len(), - expected: bytes as usize, - }) - } else { - let mut value: u32 = 0; - let mut shift = 0; - for x in &self.dict_id { - value |= u32::from(*x) << shift; - shift += 8; - } - - Ok(Some(value)) - } - } + pub fn dictionary_id(&self) -> Option { + self.dict_id } - pub fn frame_content_size(&self) -> Result { - let bytes = self.descriptor.frame_content_size_bytes()?; - - if self.frame_content_size.len() != (bytes as usize) { - return Err(FrameHeaderError::MismatchedFrameSize { - got: self.frame_content_size.len(), - expected: bytes, - }); - } - - match bytes { - 0 => Err(FrameHeaderError::FrameSizeIsZero), - 1 => Ok(u64::from(self.frame_content_size[0])), - 2 => { - let val = (u64::from(self.frame_content_size[1]) << 8) - + (u64::from(self.frame_content_size[0])); - Ok(val + 256) //this weird offset is from the documentation. Only if bytes == 2 - } - 4 => { - let val = self.frame_content_size[..4] - .try_into() - .expect("optimized away"); - let val = u32::from_le_bytes(val); - Ok(u64::from(val)) - } - 8 => { - let val = self.frame_content_size[..8] - .try_into() - .expect("optimized away"); - let val = u64::from_le_bytes(val); - Ok(val) - } - other => Err(FrameHeaderError::InvalidFrameSize { got: other }), - } - } -} - -#[derive(Debug, thiserror::Error)] -#[non_exhaustive] -pub enum FrameCheckError { - #[error("magic_num wrong. Is: {got}. Should be: {MAGIC_NUM}")] - WrongMagicNum { got: u32 }, - #[error("Reserved Flag set. Must be zero")] - ReservedFlagSet, - #[error(transparent)] - FrameHeaderError(#[from] FrameHeaderError), -} - -impl Frame { - pub fn check_valid(&self) -> Result<(), FrameCheckError> { - if self.magic_num != MAGIC_NUM { - Err(FrameCheckError::WrongMagicNum { - got: self.magic_num, - }) - } else if self.header.descriptor.reserved_flag() { - Err(FrameCheckError::ReservedFlagSet) - } else { - self.header.dictionary_id()?; - self.header.window_size()?; - if self.header.descriptor.single_segment_flag() { - self.header.frame_content_size()?; - } - Ok(()) - } + pub fn frame_content_size(&self) -> u64 { + self.frame_content_size } } @@ -214,27 +130,29 @@ impl Frame { #[non_exhaustive] pub enum ReadFrameHeaderError { #[error("Error while reading magic number: {0}")] - MagicNumberReadError(#[source] io::Error), + MagicNumberReadError(#[source] Error), + #[error("Read wrong magic number: 0x{0:X}")] + BadMagicNumber(u32), #[error("Error while reading frame descriptor: {0}")] - FrameDescriptorReadError(#[source] io::Error), + FrameDescriptorReadError(#[source] Error), #[error(transparent)] InvalidFrameDescriptor(#[from] FrameDescriptorError), #[error("Error while reading window descriptor: {0}")] - WindowDescriptorReadError(#[source] io::Error), + WindowDescriptorReadError(#[source] Error), #[error("Error while reading dictionary id: {0}")] - DictionaryIdReadError(#[source] io::Error), + DictionaryIdReadError(#[source] Error), #[error("Error while reading frame content size: {0}")] - FrameContentSizeReadError(#[source] io::Error), + FrameContentSizeReadError(#[source] Error), #[error("SkippableFrame encountered with MagicNumber 0x{0:X} and length {1} bytes")] SkipFrame(u32, u32), } -use std::io::Read; pub fn read_frame_header(mut r: impl Read) -> Result<(Frame, u8), ReadFrameHeaderError> { use ReadFrameHeaderError as err; let mut buf = [0u8; 4]; r.read_exact(&mut buf).map_err(err::MagicNumberReadError)?; + let mut bytes_read = 4; let magic_num = u32::from_le_bytes(buf); // Skippable frames have a magic number in this interval @@ -245,7 +163,9 @@ pub fn read_frame_header(mut r: impl Read) -> Result<(Frame, u8), ReadFrameHeade return Err(ReadFrameHeaderError::SkipFrame(magic_num, skip_size)); } - let mut bytes_read = 4; + if magic_num != MAGIC_NUM { + return Err(ReadFrameHeaderError::BadMagicNumber(magic_num)); + } r.read_exact(&mut buf[0..1]) .map_err(err::FrameDescriptorReadError)?; @@ -255,8 +175,8 @@ pub fn read_frame_header(mut r: impl Read) -> Result<(Frame, u8), ReadFrameHeade let mut frame_header = FrameHeader { descriptor: FrameDescriptor(desc.0), - dict_id: vec![0; desc.dictionary_id_bytes()? as usize], - frame_content_size: vec![0; desc.frame_content_size_bytes()? as usize], + dict_id: None, + frame_content_size: 0, window_descriptor: 0, }; @@ -267,20 +187,42 @@ pub fn read_frame_header(mut r: impl Read) -> Result<(Frame, u8), ReadFrameHeade bytes_read += 1; } - if !frame_header.dict_id.is_empty() { - r.read_exact(frame_header.dict_id.as_mut_slice()) - .map_err(err::DictionaryIdReadError)?; - bytes_read += frame_header.dict_id.len(); + let dict_id_len = desc.dictionary_id_bytes()? as usize; + if dict_id_len != 0 { + let buf = &mut buf[..dict_id_len]; + r.read_exact(buf).map_err(err::DictionaryIdReadError)?; + bytes_read += dict_id_len; + let mut dict_id = 0u32; + + #[allow(clippy::needless_range_loop)] + for i in 0..dict_id_len { + dict_id += (buf[i] as u32) << (8 * i); + } + if dict_id != 0 { + frame_header.dict_id = Some(dict_id); + } } - if !frame_header.frame_content_size.is_empty() { - r.read_exact(frame_header.frame_content_size.as_mut_slice()) + let fcs_len = desc.frame_content_size_bytes()? as usize; + if fcs_len != 0 { + let mut fcs_buf = [0u8; 8]; + let fcs_buf = &mut fcs_buf[..fcs_len]; + r.read_exact(fcs_buf) .map_err(err::FrameContentSizeReadError)?; - bytes_read += frame_header.frame_content_size.len(); + bytes_read += fcs_len; + let mut fcs = 0u64; + + #[allow(clippy::needless_range_loop)] + for i in 0..fcs_len { + fcs += (fcs_buf[i] as u64) << (8 * i); + } + if fcs_len == 2 { + fcs += 256; + } + frame_header.frame_content_size = fcs; } let frame: Frame = Frame { - magic_num, header: frame_header, }; diff --git a/vendor/ruzstd/src/frame_decoder.rs b/vendor/ruzstd/src/frame_decoder.rs index 560e82810..18af6b3d3 100644 --- a/vendor/ruzstd/src/frame_decoder.rs +++ b/vendor/ruzstd/src/frame_decoder.rs @@ -2,10 +2,11 @@ use super::frame; use crate::decoding::dictionary::Dictionary; use crate::decoding::scratch::DecoderScratch; use crate::decoding::{self, dictionary}; -use std::collections::HashMap; -use std::convert::TryInto; -use std::hash::Hasher; -use std::io::{self, Read}; +use crate::io::{Error, Read, Write}; +use alloc::collections::BTreeMap; +use alloc::vec::Vec; +use core::convert::TryInto; +use core::hash::Hasher; /// This implements a decoder for zstd frames. This decoder is able to decode frames only partially and gives control /// over how many bytes/blocks will be decoded at a time (so you don't have to decode a 10GB file into memory all at once). @@ -17,11 +18,15 @@ use std::io::{self, Read}; /// Workflow is as follows: /// ``` /// use ruzstd::frame_decoder::BlockDecodingStrategy; -/// use std::io::Read; -/// use std::io::Write; /// +/// # #[cfg(feature = "std")] +/// use std::io::{Read, Write}; /// -/// fn decode_this(mut file: impl std::io::Read) { +/// // no_std environments can use the crate's own Read traits +/// # #[cfg(not(feature = "std"))] +/// use ruzstd::io::{Read, Write}; +/// +/// fn decode_this(mut file: impl Read) { /// //Create a new decoder /// let mut frame_dec = ruzstd::FrameDecoder::new(); /// let mut result = Vec::new(); @@ -50,12 +55,13 @@ use std::io::{self, Read}; /// } /// /// fn do_something(data: &[u8]) { +/// # #[cfg(feature = "std")] /// std::io::stdout().write_all(data).unwrap(); /// } /// ``` pub struct FrameDecoder { state: Option, - dicts: HashMap, + dicts: BTreeMap, } struct FrameDecoderState { @@ -81,8 +87,6 @@ pub enum FrameDecoderError { ReadFrameHeaderError(#[from] frame::ReadFrameHeaderError), #[error(transparent)] FrameHeaderError(#[from] frame::FrameHeaderError), - #[error(transparent)] - FrameCheckError(#[from] frame::FrameCheckError), #[error("Specified window_size is too big; Requested: {requested}, Max: {MAX_WINDOW_SIZE}")] WindowSizeTooBig { requested: u64 }, #[error(transparent)] @@ -92,17 +96,17 @@ pub enum FrameDecoderError { #[error("Failed to parse block header: {0}")] FailedToReadBlockBody(decoding::block_decoder::DecodeBlockContentError), #[error("Failed to read checksum: {0}")] - FailedToReadChecksum(#[source] io::Error), + FailedToReadChecksum(#[source] Error), #[error("Decoder must initialized or reset before using it")] NotYetInitialized, #[error("Decoder encountered error while initializing: {0}")] FailedToInitialize(frame::FrameHeaderError), #[error("Decoder encountered error while draining the decodebuffer: {0}")] - FailedToDrainDecodebuffer(#[source] io::Error), + FailedToDrainDecodebuffer(#[source] Error), #[error("Target must have at least as many bytes as the contentsize of the frame reports")] TargetTooSmall, - #[error("Frame header specified dictionary id that wasnt provided by add_dict() or reset_with_dict()")] - DictNotProvided, + #[error("Frame header specified dictionary id 0x{dict_id:X} that wasnt provided by add_dict() or reset_with_dict()")] + DictNotProvided { dict_id: u32 }, } const MAX_WINDOW_SIZE: u64 = 1024 * 1024 * 100; @@ -111,7 +115,6 @@ impl FrameDecoderState { pub fn new(source: impl Read) -> Result { let (frame, header_size) = frame::read_frame_header(source)?; let window_size = frame.header.window_size()?; - frame.check_valid()?; Ok(FrameDecoderState { frame, frame_finished: false, @@ -126,7 +129,6 @@ impl FrameDecoderState { pub fn reset(&mut self, source: impl Read) -> Result<(), FrameDecoderError> { let (frame, header_size) = frame::read_frame_header(source)?; let window_size = frame.header.window_size()?; - frame.check_valid()?; if window_size > MAX_WINDOW_SIZE { return Err(FrameDecoderError::WindowSizeTooBig { @@ -158,7 +160,7 @@ impl FrameDecoder { pub fn new() -> FrameDecoder { FrameDecoder { state: None, - dicts: HashMap::new(), + dicts: BTreeMap::new(), } } @@ -171,14 +173,6 @@ impl FrameDecoder { pub fn init(&mut self, source: impl Read) -> Result<(), FrameDecoderError> { self.reset(source) } - /// Like init but provides the dict to use for the next frame - pub fn init_with_dict( - &mut self, - source: impl Read, - dict: &[u8], - ) -> Result<(), FrameDecoderError> { - self.reset_with_dict(source, dict) - } /// reset() will allocate all needed buffers if it is the first time this decoder is used /// else they just reset these buffers with not further allocations @@ -187,46 +181,55 @@ impl FrameDecoder { /// /// equivalent to init() pub fn reset(&mut self, source: impl Read) -> Result<(), FrameDecoderError> { - match &mut self.state { - Some(s) => s.reset(source), + use FrameDecoderError as err; + let state = match &mut self.state { + Some(s) => { + s.reset(source)?; + s + } None => { self.state = Some(FrameDecoderState::new(source)?); - Ok(()) + self.state.as_mut().unwrap() } - } - } - - /// Like reset but provides the dict to use for the next frame - pub fn reset_with_dict( - &mut self, - source: impl Read, - dict: &[u8], - ) -> Result<(), FrameDecoderError> { - self.reset(source)?; - if let Some(state) = &mut self.state { - let id = state.decoder_scratch.load_dict(dict)?; - state.using_dict = Some(id); }; + if let Some(dict_id) = state.frame.header.dictionary_id() { + let dict = self + .dicts + .get(&dict_id) + .ok_or(err::DictNotProvided { dict_id })?; + state.decoder_scratch.init_from_dict(dict); + state.using_dict = Some(dict_id); + } Ok(()) } /// Add a dict to the FrameDecoder that can be used when needed. The FrameDecoder uses the appropriate one dynamically - pub fn add_dict(&mut self, raw_dict: &[u8]) -> Result<(), FrameDecoderError> { - let dict = Dictionary::decode_dict(raw_dict)?; + pub fn add_dict(&mut self, dict: Dictionary) -> Result<(), FrameDecoderError> { self.dicts.insert(dict.id, dict); Ok(()) } - /// Returns how many bytes the frame contains after decompression - pub fn content_size(&self) -> Option { - let state = match &self.state { - None => return Some(0), - Some(s) => s, + pub fn force_dict(&mut self, dict_id: u32) -> Result<(), FrameDecoderError> { + use FrameDecoderError as err; + let Some(state) = self.state.as_mut() else { + return Err(err::NotYetInitialized); }; - match state.frame.header.frame_content_size() { - Err(_) => None, - Ok(x) => Some(x), + let dict = self + .dicts + .get(&dict_id) + .ok_or(err::DictNotProvided { dict_id })?; + state.decoder_scratch.init_from_dict(dict); + state.using_dict = Some(dict_id); + + Ok(()) + } + + /// Returns how many bytes the frame contains after decompression + pub fn content_size(&self) -> u64 { + match &self.state { + None => 0, + Some(s) => s.frame.header.frame_content_size(), } } @@ -297,47 +300,26 @@ impl FrameDecoder { use FrameDecoderError as err; let state = self.state.as_mut().ok_or(err::NotYetInitialized)?; - if let Some(id) = state.frame.header.dictionary_id().map_err( - //should never happen we check this directly after decoding the frame header - err::FailedToInitialize, - )? { - match state.using_dict { - Some(using_id) => { - //happy - debug_assert!(id == using_id); - } - None => { - let dict = self.dicts.get(&id).ok_or(err::DictNotProvided)?; - state.decoder_scratch.use_dict(dict); - state.using_dict = Some(id); - } - } - } - let mut block_dec = decoding::block_decoder::new(); let buffer_size_before = state.decoder_scratch.buffer.len(); let block_counter_before = state.block_counter; loop { - if crate::VERBOSE { - println!("################"); - println!("Next Block: {}", state.block_counter); - println!("################"); - } + vprintln!("################"); + vprintln!("Next Block: {}", state.block_counter); + vprintln!("################"); let (block_header, block_header_size) = block_dec .read_block_header(&mut source) .map_err(err::FailedToReadBlockHeader)?; state.bytes_read_counter += u64::from(block_header_size); - if crate::VERBOSE { - println!(); - println!( - "Found {} block with size: {}, which will be of size: {}", - block_header.block_type, - block_header.content_size, - block_header.decompressed_size - ); - } + vprintln!(); + vprintln!( + "Found {} block with size: {}, which will be of size: {}", + block_header.block_type, + block_header.content_size, + block_header.decompressed_size + ); let bytes_read_in_block_body = block_dec .decode_block_content(&block_header, &mut state.decoder_scratch, &mut source) @@ -346,9 +328,7 @@ impl FrameDecoder { state.block_counter += 1; - if crate::VERBOSE { - println!("Output: {}", state.decoder_scratch.buffer.len()); - } + vprintln!("Output: {}", state.decoder_scratch.buffer.len()); if block_header.last_block { state.frame_finished = true; @@ -396,7 +376,7 @@ impl FrameDecoder { /// Collect bytes and retain window_size bytes while decoding is still going on. /// After decoding of the frame (is_finished() == true) has finished it will collect all remaining bytes - pub fn collect_to_writer(&mut self, w: impl std::io::Write) -> Result { + pub fn collect_to_writer(&mut self, w: impl Write) -> Result { let finished = self.is_finished(); let state = match &mut self.state { None => return Ok(0), @@ -482,23 +462,6 @@ impl FrameDecoder { return Ok((4, 0)); } - if let Some(id) = state.frame.header.dictionary_id().map_err( - //should never happen we check this directly after decoding the frame header - err::FailedToInitialize, - )? { - match state.using_dict { - Some(using_id) => { - //happy - debug_assert!(id == using_id); - } - None => { - let dict = self.dicts.get(&id).ok_or(err::DictNotProvided)?; - state.decoder_scratch.use_dict(dict); - state.using_dict = Some(id); - } - } - } - loop { //check if there are enough bytes for the next header if mt_source.len() < 3 { @@ -554,8 +517,8 @@ impl FrameDecoder { /// Read bytes from the decode_buffer that are no longer needed. While the frame is not yet finished /// this will retain window_size bytes, else it will drain it completely -impl std::io::Read for FrameDecoder { - fn read(&mut self, target: &mut [u8]) -> std::result::Result { +impl Read for FrameDecoder { + fn read(&mut self, target: &mut [u8]) -> Result { let state = match &mut self.state { None => return Ok(0), Some(s) => s, diff --git a/vendor/ruzstd/src/fse/fse_decoder.rs b/vendor/ruzstd/src/fse/fse_decoder.rs index 1847da728..21868ff84 100644 --- a/vendor/ruzstd/src/fse/fse_decoder.rs +++ b/vendor/ruzstd/src/fse/fse_decoder.rs @@ -1,7 +1,7 @@ use crate::decoding::bit_reader::BitReader; use crate::decoding::bit_reader_reverse::{BitReaderReversed, GetBitsError}; +use alloc::vec::Vec; -#[derive(Clone)] pub struct FSETable { pub decode: Vec, //used to decode symbols, and calculate the next state @@ -113,6 +113,15 @@ impl FSETable { } } + pub fn reinit_from(&mut self, other: &Self) { + self.reset(); + self.symbol_counter.extend_from_slice(&other.symbol_counter); + self.symbol_probabilities + .extend_from_slice(&other.symbol_probabilities); + self.decode.extend_from_slice(&other.decode); + self.accuracy_log = other.accuracy_log; + } + pub fn reset(&mut self) { self.symbol_counter.clear(); self.symbol_probabilities.clear(); diff --git a/vendor/ruzstd/src/huff0/huff0_decoder.rs b/vendor/ruzstd/src/huff0/huff0_decoder.rs index 831ddd69c..a8aa74563 100644 --- a/vendor/ruzstd/src/huff0/huff0_decoder.rs +++ b/vendor/ruzstd/src/huff0/huff0_decoder.rs @@ -1,7 +1,7 @@ use crate::decoding::bit_reader_reverse::{BitReaderReversed, GetBitsError}; use crate::fse::{FSEDecoder, FSEDecoderError, FSETable, FSETableError}; +use alloc::vec::Vec; -#[derive(Clone)] pub struct HuffmanTable { decode: Vec, @@ -134,6 +134,16 @@ impl HuffmanTable { } } + pub fn reinit_from(&mut self, other: &Self) { + self.reset(); + self.decode.extend_from_slice(&other.decode); + self.weights.extend_from_slice(&other.weights); + self.max_num_bits = other.max_num_bits; + self.bits.extend_from_slice(&other.bits); + self.rank_indexes.extend_from_slice(&other.rank_indexes); + self.fse_table.reinit_from(&other.fse_table); + } + pub fn reset(&mut self) { self.decode.clear(); self.weights.clear(); @@ -182,12 +192,10 @@ impl HuffmanTable { }); } - if crate::VERBOSE { - println!( - "Building fse table for huffman weights used: {}", - bytes_used_by_fse_header - ); - } + vprintln!( + "Building fse table for huffman weights used: {}", + bytes_used_by_fse_header + ); let mut dec1 = FSEDecoder::new(&self.fse_table); let mut dec2 = FSEDecoder::new(&self.fse_table); diff --git a/vendor/ruzstd/src/io.rs b/vendor/ruzstd/src/io.rs new file mode 100644 index 000000000..6970cd132 --- /dev/null +++ b/vendor/ruzstd/src/io.rs @@ -0,0 +1,2 @@ +#[cfg(feature = "std")] +pub use std::io::{Error, ErrorKind, Read, Write}; diff --git a/vendor/ruzstd/src/io_nostd.rs b/vendor/ruzstd/src/io_nostd.rs new file mode 100644 index 000000000..1e5d14166 --- /dev/null +++ b/vendor/ruzstd/src/io_nostd.rs @@ -0,0 +1,152 @@ +use alloc::boxed::Box; + +#[non_exhaustive] +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Ord, PartialOrd)] +pub enum ErrorKind { + Interrupted, + UnexpectedEof, + WouldBlock, + Other, +} + +impl ErrorKind { + fn as_str(&self) -> &'static str { + use ErrorKind::*; + match *self { + Interrupted => "operation interrupted", + UnexpectedEof => "unexpected end of file", + WouldBlock => "operation would block", + Other => "other error", + } + } +} + +impl core::fmt::Display for ErrorKind { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(self.as_str()) + } +} + +#[derive(Debug)] +pub struct Error { + kind: ErrorKind, + err: Option>, +} + +impl Error { + pub fn new(kind: ErrorKind, err: E) -> Self + where + E: Into>, + { + Self { + kind, + err: Some(err.into()), + } + } + + pub fn from(kind: ErrorKind) -> Self { + Self { kind, err: None } + } + + pub fn kind(&self) -> ErrorKind { + self.kind + } + + pub fn get_ref(&self) -> Option<&(dyn core::error::Error + Send + Sync + 'static)> { + self.err.as_ref().map(|e| e.as_ref()) + } + + pub fn get_mut(&mut self) -> Option<&mut (dyn core::error::Error + Send + Sync + 'static)> { + self.err.as_mut().map(|e| e.as_mut()) + } + + pub fn into_inner(self) -> Option> { + self.err + } +} + +impl core::fmt::Display for Error { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + f.write_str(self.kind.as_str())?; + + if let Some(ref e) = self.err { + e.fmt(f)?; + } + + Ok(()) + } +} + +impl core::error::Error for Error {} + +impl From for Error { + fn from(value: ErrorKind) -> Self { + Self::from(value) + } +} + +pub trait Read { + fn read(&mut self, buf: &mut [u8]) -> Result; + + fn read_exact(&mut self, mut buf: &mut [u8]) -> Result<(), Error> { + while !buf.is_empty() { + match self.read(buf) { + Ok(0) => break, + Ok(n) => { + let tmp = buf; + buf = &mut tmp[n..]; + } + Err(ref e) if e.kind() == ErrorKind::Interrupted => {} + Err(e) => return Err(e), + } + } + if !buf.is_empty() { + Err(Error::from(ErrorKind::UnexpectedEof)) + } else { + Ok(()) + } + } +} + +impl Read for &[u8] { + fn read(&mut self, buf: &mut [u8]) -> Result { + let size = core::cmp::min(self.len(), buf.len()); + let (to_copy, rest) = self.split_at(size); + + if size == 1 { + buf[0] = to_copy[0]; + } else { + buf[..size].copy_from_slice(to_copy); + } + + *self = rest; + Ok(size) + } +} + +impl<'a, T> Read for &'a mut T +where + T: Read, +{ + fn read(&mut self, buf: &mut [u8]) -> Result { + (*self).read(buf) + } +} + +pub trait Write { + fn write(&mut self, buf: &[u8]) -> Result; + fn flush(&mut self) -> Result<(), Error>; +} + +impl<'a, T> Write for &'a mut T +where + T: Write, +{ + fn write(&mut self, buf: &[u8]) -> Result { + (*self).write(buf) + } + + fn flush(&mut self) -> Result<(), Error> { + (*self).flush() + } +} diff --git a/vendor/ruzstd/src/lib.rs b/vendor/ruzstd/src/lib.rs index e079ad31b..aecef623b 100644 --- a/vendor/ruzstd/src/lib.rs +++ b/vendor/ruzstd/src/lib.rs @@ -1,4 +1,23 @@ +#![no_std] #![deny(trivial_casts, trivial_numeric_casts, rust_2018_idioms)] +#![cfg_attr(not(feature = "std"), feature(error_in_core))] + +#[cfg(feature = "std")] +extern crate std; + +extern crate alloc; + +#[cfg(feature = "std")] +pub const VERBOSE: bool = false; + +macro_rules! vprintln { + ($($x:expr),*) => { + #[cfg(feature = "std")] + if crate::VERBOSE { + std::println!($($x),*); + } + } +} pub mod blocks; pub mod decoding; @@ -9,7 +28,15 @@ pub mod huff0; pub mod streaming_decoder; mod tests; -pub const VERBOSE: bool = false; +#[cfg(feature = "std")] +pub mod io; + +#[cfg(not(feature = "std"))] +pub mod io_nostd; + +#[cfg(not(feature = "std"))] +pub use io_nostd as io; + pub use frame_decoder::BlockDecodingStrategy; pub use frame_decoder::FrameDecoder; pub use streaming_decoder::StreamingDecoder; diff --git a/vendor/ruzstd/src/streaming_decoder.rs b/vendor/ruzstd/src/streaming_decoder.rs index 2613a363b..f345a897c 100644 --- a/vendor/ruzstd/src/streaming_decoder.rs +++ b/vendor/ruzstd/src/streaming_decoder.rs @@ -1,27 +1,33 @@ +use core::borrow::BorrowMut; + use crate::frame_decoder::{BlockDecodingStrategy, FrameDecoder, FrameDecoderError}; -use std::io::Read; +use crate::io::{Error, ErrorKind, Read}; /// High level decoder that implements a io::Read that can be used with /// io::Read::read_to_end / io::Read::read_exact or passing this to another library / module as a source for the decoded content /// /// The lower level FrameDecoder by comparison allows for finer grained control but need sto have it's decode_blocks method called continously /// to decode the zstd-frame. -pub struct StreamingDecoder { - pub decoder: FrameDecoder, +pub struct StreamingDecoder> { + pub decoder: DEC, source: READ, } -impl StreamingDecoder { - pub fn new(mut source: READ) -> Result, FrameDecoderError> { - let mut decoder = FrameDecoder::new(); - decoder.init(&mut source)?; +impl> StreamingDecoder { + pub fn new_with_decoder( + mut source: READ, + mut decoder: DEC, + ) -> Result, FrameDecoderError> { + decoder.borrow_mut().init(&mut source)?; Ok(StreamingDecoder { decoder, source }) } +} - pub fn new_with_decoder( +impl StreamingDecoder { + pub fn new( mut source: READ, - mut decoder: FrameDecoder, - ) -> Result, FrameDecoderError> { + ) -> Result, FrameDecoderError> { + let mut decoder = FrameDecoder::new(); decoder.init(&mut source)?; Ok(StreamingDecoder { decoder, source }) } @@ -31,9 +37,10 @@ impl StreamingDecoder { } } -impl Read for StreamingDecoder { - fn read(&mut self, buf: &mut [u8]) -> std::io::Result { - if self.decoder.is_finished() && self.decoder.can_collect() == 0 { +impl> Read for StreamingDecoder { + fn read(&mut self, buf: &mut [u8]) -> Result { + let decoder = self.decoder.borrow_mut(); + if decoder.is_finished() && decoder.can_collect() == 0 { //No more bytes can ever be decoded return Ok(0); } @@ -43,24 +50,24 @@ impl Read for StreamingDecoder { // So we need to call this until we can actually collect enough bytes // TODO add BlockDecodingStrategy::UntilCollectable(usize) that pushes this logic into the decode_blocks function - while self.decoder.can_collect() < buf.len() && !self.decoder.is_finished() { + while decoder.can_collect() < buf.len() && !decoder.is_finished() { //More bytes can be decoded - let additional_bytes_needed = buf.len() - self.decoder.can_collect(); - match self.decoder.decode_blocks( + let additional_bytes_needed = buf.len() - decoder.can_collect(); + match decoder.decode_blocks( &mut self.source, BlockDecodingStrategy::UptoBytes(additional_bytes_needed), ) { Ok(_) => { /*Nothing to do*/ } Err(e) => { - let err = std::io::Error::new( - std::io::ErrorKind::Other, - format!("Error in the zstd decoder: {:?}", e), + let err = Error::new( + ErrorKind::Other, + alloc::format!("Error in the zstd decoder: {:?}", e), ); return Err(err); } } } - self.decoder.read(buf) + decoder.read(buf) } } diff --git a/vendor/ruzstd/src/tests/decode_corpus.rs b/vendor/ruzstd/src/tests/decode_corpus.rs index ea1a723a3..a8e21683b 100644 --- a/vendor/ruzstd/src/tests/decode_corpus.rs +++ b/vendor/ruzstd/src/tests/decode_corpus.rs @@ -1,8 +1,13 @@ #[test] fn test_decode_corpus_files() { + extern crate std; use crate::frame_decoder; + use alloc::borrow::ToOwned; + use alloc::string::{String, ToString}; + use alloc::vec::Vec; use std::fs; use std::io::Read; + use std::println; let mut success_counter = 0; let mut fail_counter_diff = 0; diff --git a/vendor/ruzstd/src/tests/dict_test.rs b/vendor/ruzstd/src/tests/dict_test.rs index a8a5fd46b..5bf76f91e 100644 --- a/vendor/ruzstd/src/tests/dict_test.rs +++ b/vendor/ruzstd/src/tests/dict_test.rs @@ -1,6 +1,7 @@ #[test] fn test_dict_parsing() { use crate::decoding::dictionary::Dictionary; + use alloc::vec; let mut raw = vec![0u8; 8]; // correct magic num @@ -75,9 +76,14 @@ fn test_dict_parsing() { #[test] fn test_dict_decoding() { + extern crate std; use crate::frame_decoder; + use alloc::borrow::ToOwned; + use alloc::string::{String, ToString}; + use alloc::vec::Vec; use std::fs; use std::io::Read; + use std::println; let mut success_counter = 0; let mut fail_counter_diff = 0; @@ -99,7 +105,8 @@ fn test_dict_decoding() { }); let mut frame_dec = frame_decoder::FrameDecoder::new(); - frame_dec.add_dict(&dict).unwrap(); + let dict = crate::decoding::dictionary::Dictionary::decode_dict(&dict).unwrap(); + frame_dec.add_dict(dict).unwrap(); for file in files { let f = file.unwrap(); diff --git a/vendor/ruzstd/src/tests/fuzz_regressions.rs b/vendor/ruzstd/src/tests/fuzz_regressions.rs index 2e293af4d..bc675cad3 100644 --- a/vendor/ruzstd/src/tests/fuzz_regressions.rs +++ b/vendor/ruzstd/src/tests/fuzz_regressions.rs @@ -1,6 +1,8 @@ #[test] fn test_all_artifacts() { + extern crate std; use crate::frame_decoder; + use std::borrow::ToOwned; use std::fs; use std::fs::File; diff --git a/vendor/ruzstd/src/tests/mod.rs b/vendor/ruzstd/src/tests/mod.rs index 070565c36..0f25073e5 100644 --- a/vendor/ruzstd/src/tests/mod.rs +++ b/vendor/ruzstd/src/tests/mod.rs @@ -1,4 +1,28 @@ #[cfg(test)] +use alloc::vec; + +#[cfg(test)] +use alloc::vec::Vec; + +#[cfg(test)] +extern crate std; + +#[cfg(all(test, not(feature = "std")))] +impl crate::io_nostd::Read for std::fs::File { + fn read(&mut self, buf: &mut [u8]) -> Result { + std::io::Read::read(self, buf).map_err(|e| { + if e.get_ref().is_none() { + crate::io_nostd::Error::from(crate::io_nostd::ErrorKind::Other) + } else { + crate::io_nostd::Error::new( + crate::io_nostd::ErrorKind::Other, + e.into_inner().unwrap(), + ) + } + }) + } +} + #[test] fn skippable_frame() { use crate::frame; @@ -34,8 +58,7 @@ fn test_frame_header_reading() { use std::fs; let mut content = fs::File::open("./decodecorpus_files/z000088.zst").unwrap(); - let (frame, _) = frame::read_frame_header(&mut content).unwrap(); - frame.check_valid().unwrap(); + let (_frame, _) = frame::read_frame_header(&mut content).unwrap(); } #[test] @@ -45,8 +68,7 @@ fn test_block_header_reading() { use std::fs; let mut content = fs::File::open("./decodecorpus_files/z000088.zst").unwrap(); - let (frame, _) = frame::read_frame_header(&mut content).unwrap(); - frame.check_valid().unwrap(); + let (_frame, _) = frame::read_frame_header(&mut content).unwrap(); let mut block_dec = decoding::block_decoder::new(); let block_header = block_dec.read_block_header(&mut content).unwrap(); @@ -131,16 +153,16 @@ fn test_decode_from_to() { match frame_dec.get_checksum_from_data() { Some(chksum) => { if frame_dec.get_calculated_checksum().unwrap() != chksum { - println!( + std::println!( "Checksum did not match! From data: {}, calculated while decoding: {}\n", chksum, frame_dec.get_calculated_checksum().unwrap() ); } else { - println!("Checksums are ok!\n"); + std::println!("Checksums are ok!\n"); } } - None => println!("No checksums to test\n"), + None => std::println!("No checksums to test\n"), } let original_f = File::open("./decodecorpus_files/z000088").unwrap(); @@ -163,7 +185,7 @@ fn test_decode_from_to() { for idx in 0..min { if original[idx] != result[idx] { counter += 1; - //println!( + //std::println!( // "Original {:3} not equal to result {:3} at byte: {}", // original[idx], result[idx], idx, //); @@ -204,10 +226,10 @@ fn test_specific_file() { let original_f = fs::File::open("./decodecorpus_files/z000088").unwrap(); let original: Vec = original_f.bytes().map(|x| x.unwrap()).collect(); - println!("Results for file: {}", path); + std::println!("Results for file: {}", path); if original.len() != result.len() { - println!( + std::println!( "Result has wrong length: {}, should be: {}", result.len(), original.len() @@ -223,18 +245,19 @@ fn test_specific_file() { for idx in 0..min { if original[idx] != result[idx] { counter += 1; - //println!( + //std::println!( // "Original {:3} not equal to result {:3} at byte: {}", // original[idx], result[idx], idx, //); } } if counter > 0 { - println!("Result differs in at least {} bytes from original", counter); + std::println!("Result differs in at least {} bytes from original", counter); } } #[test] +#[cfg(feature = "std")] fn test_streaming() { use std::fs; use std::io::Read; @@ -265,7 +288,7 @@ fn test_streaming() { for idx in 0..min { if original[idx] != result[idx] { counter += 1; - //println!( + //std::println!( // "Original {:3} not equal to result {:3} at byte: {}", // original[idx], result[idx], idx, //); @@ -288,7 +311,91 @@ fn test_streaming() { let original_f = fs::File::open("./decodecorpus_files/z000068").unwrap(); let original: Vec = original_f.bytes().map(|x| x.unwrap()).collect(); - println!("Results for file:"); + std::println!("Results for file:"); + + if original.len() != result.len() { + panic!( + "Result has wrong length: {}, should be: {}", + result.len(), + original.len() + ); + } + + let mut counter = 0; + let min = if original.len() < result.len() { + original.len() + } else { + result.len() + }; + for idx in 0..min { + if original[idx] != result[idx] { + counter += 1; + //std::println!( + // "Original {:3} not equal to result {:3} at byte: {}", + // original[idx], result[idx], idx, + //); + } + } + if counter > 0 { + panic!("Result differs in at least {} bytes from original", counter); + } +} + +#[test] +#[cfg(not(feature = "std"))] +fn test_streaming_no_std() { + use crate::io::Read; + + let content = include_bytes!("../../decodecorpus_files/z000088.zst"); + let mut content = content.as_slice(); + let mut stream = crate::streaming_decoder::StreamingDecoder::new(&mut content).unwrap(); + + let original = include_bytes!("../../decodecorpus_files/z000088"); + let mut result = Vec::new(); + result.resize(original.len(), 0); + Read::read_exact(&mut stream, &mut result).unwrap(); + + if original.len() != result.len() { + panic!( + "Result has wrong length: {}, should be: {}", + result.len(), + original.len() + ); + } + + let mut counter = 0; + let min = if original.len() < result.len() { + original.len() + } else { + result.len() + }; + for idx in 0..min { + if original[idx] != result[idx] { + counter += 1; + //std::println!( + // "Original {:3} not equal to result {:3} at byte: {}", + // original[idx], result[idx], idx, + //); + } + } + if counter > 0 { + panic!("Result differs in at least {} bytes from original", counter); + } + + // Test resetting to a new file while keeping the old decoder + + let content = include_bytes!("../../decodecorpus_files/z000068.zst"); + let mut content = content.as_slice(); + let mut stream = + crate::streaming_decoder::StreamingDecoder::new_with_decoder(&mut content, stream.inner()) + .unwrap(); + + let original = include_bytes!("../../decodecorpus_files/z000068"); + let mut result = Vec::new(); + result.resize(original.len(), 0); + Read::read_exact(&mut stream, &mut result).unwrap(); + + std::println!("Results for file:"); if original.len() != result.len() { panic!( @@ -307,7 +414,7 @@ fn test_streaming() { for idx in 0..min { if original[idx] != result[idx] { counter += 1; - //println!( + //std::println!( // "Original {:3} not equal to result {:3} at byte: {}", // original[idx], result[idx], idx, //); -- cgit v1.2.3