use std::cmp; use std::io; use std::io::prelude::*; use std::mem; use super::{GzBuilder, GzHeader}; use super::{FCOMMENT, FEXTRA, FHCRC, FNAME}; use crate::crc::{Crc, CrcReader}; use crate::deflate; use crate::Compression; fn copy(into: &mut [u8], from: &[u8], pos: &mut usize) -> usize { let min = cmp::min(into.len(), from.len() - *pos); for (slot, val) in into.iter_mut().zip(from[*pos..*pos + min].iter()) { *slot = *val; } *pos += min; min } pub(crate) fn corrupt() -> io::Error { io::Error::new( io::ErrorKind::InvalidInput, "corrupt gzip stream does not have a matching checksum", ) } fn bad_header() -> io::Error { io::Error::new(io::ErrorKind::InvalidInput, "invalid gzip header") } fn read_le_u16(r: &mut Buffer) -> io::Result { let mut b = [0; 2]; r.read_and_forget(&mut b)?; Ok((b[0] as u16) | ((b[1] as u16) << 8)) } fn read_gz_header_part<'a, R: Read>(r: &'a mut Buffer<'a, R>) -> io::Result<()> { loop { match r.part.state { GzHeaderParsingState::Start => { let mut header = [0; 10]; r.read_and_forget(&mut header)?; if header[0] != 0x1f || header[1] != 0x8b { return Err(bad_header()); } if header[2] != 8 { return Err(bad_header()); } r.part.flg = header[3]; r.part.header.mtime = ((header[4] as u32) << 0) | ((header[5] as u32) << 8) | ((header[6] as u32) << 16) | ((header[7] as u32) << 24); let _xfl = header[8]; r.part.header.operating_system = header[9]; r.part.state = GzHeaderParsingState::Xlen; } GzHeaderParsingState::Xlen => { if r.part.flg & FEXTRA != 0 { r.part.xlen = read_le_u16(r)?; } r.part.state = GzHeaderParsingState::Extra; } GzHeaderParsingState::Extra => { if r.part.flg & FEXTRA != 0 { let mut extra = vec![0; r.part.xlen as usize]; r.read_and_forget(&mut extra)?; r.part.header.extra = Some(extra); } r.part.state = GzHeaderParsingState::Filename; } GzHeaderParsingState::Filename => { if r.part.flg & FNAME != 0 { if None == r.part.header.filename { r.part.header.filename = Some(Vec::new()); }; for byte in r.bytes() { let byte = byte?; if byte == 0 { break; } } } r.part.state = GzHeaderParsingState::Comment; } GzHeaderParsingState::Comment => { if r.part.flg & FCOMMENT != 0 { if None == r.part.header.comment { r.part.header.comment = Some(Vec::new()); }; for byte in r.bytes() { let byte = byte?; if byte == 0 { break; } } } r.part.state = GzHeaderParsingState::Crc; } GzHeaderParsingState::Crc => { if r.part.flg & FHCRC != 0 { let stored_crc = read_le_u16(r)?; let calced_crc = r.part.crc.sum() as u16; if stored_crc != calced_crc { return Err(corrupt()); } } return Ok(()); } } } } pub(crate) fn read_gz_header(r: &mut R) -> io::Result { let mut part = GzHeaderPartial::new(); let result = { let mut reader = Buffer::new(&mut part, r); read_gz_header_part(&mut reader) }; result.map(|()| part.take_header()) } /// A gzip streaming encoder /// /// This structure exposes a [`BufRead`] interface that will read uncompressed data /// from the underlying reader and expose the compressed version as a [`BufRead`] /// interface. /// /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples /// /// ``` /// use std::io::prelude::*; /// use std::io; /// use flate2::Compression; /// use flate2::bufread::GzEncoder; /// use std::fs::File; /// use std::io::BufReader; /// /// // Opens sample file, compresses the contents and returns a Vector or error /// // File wrapped in a BufReader implements BufRead /// /// fn open_hello_world() -> io::Result> { /// let f = File::open("examples/hello_world.txt")?; /// let b = BufReader::new(f); /// let mut gz = GzEncoder::new(b, Compression::fast()); /// let mut buffer = Vec::new(); /// gz.read_to_end(&mut buffer)?; /// Ok(buffer) /// } /// ``` #[derive(Debug)] pub struct GzEncoder { inner: deflate::bufread::DeflateEncoder>, header: Vec, pos: usize, eof: bool, } pub fn gz_encoder(header: Vec, r: R, lvl: Compression) -> GzEncoder { let crc = CrcReader::new(r); GzEncoder { inner: deflate::bufread::DeflateEncoder::new(crc, lvl), header, pos: 0, eof: false, } } impl GzEncoder { /// Creates a new encoder which will use the given compression level. /// /// The encoder is not configured specially for the emitted header. For /// header configuration, see the `GzBuilder` type. /// /// The data read from the stream `r` will be compressed and available /// through the returned reader. pub fn new(r: R, level: Compression) -> GzEncoder { GzBuilder::new().buf_read(r, level) } fn read_footer(&mut self, into: &mut [u8]) -> io::Result { if self.pos == 8 { return Ok(0); } let crc = self.inner.get_ref().crc(); let ref arr = [ (crc.sum() >> 0) as u8, (crc.sum() >> 8) as u8, (crc.sum() >> 16) as u8, (crc.sum() >> 24) as u8, (crc.amount() >> 0) as u8, (crc.amount() >> 8) as u8, (crc.amount() >> 16) as u8, (crc.amount() >> 24) as u8, ]; Ok(copy(into, arr, &mut self.pos)) } } impl GzEncoder { /// Acquires a reference to the underlying reader. pub fn get_ref(&self) -> &R { self.inner.get_ref().get_ref() } /// Acquires a mutable reference to the underlying reader. /// /// Note that mutation of the reader may result in surprising results if /// this encoder is continued to be used. pub fn get_mut(&mut self) -> &mut R { self.inner.get_mut().get_mut() } /// Returns the underlying stream, consuming this encoder pub fn into_inner(self) -> R { self.inner.into_inner().into_inner() } } #[inline] fn finish(buf: &[u8; 8]) -> (u32, u32) { let crc = ((buf[0] as u32) << 0) | ((buf[1] as u32) << 8) | ((buf[2] as u32) << 16) | ((buf[3] as u32) << 24); let amt = ((buf[4] as u32) << 0) | ((buf[5] as u32) << 8) | ((buf[6] as u32) << 16) | ((buf[7] as u32) << 24); (crc, amt) } impl Read for GzEncoder { fn read(&mut self, mut into: &mut [u8]) -> io::Result { let mut amt = 0; if self.eof { return self.read_footer(into); } else if self.pos < self.header.len() { amt += copy(into, &self.header, &mut self.pos); if amt == into.len() { return Ok(amt); } let tmp = into; into = &mut tmp[amt..]; } match self.inner.read(into)? { 0 => { self.eof = true; self.pos = 0; self.read_footer(into) } n => Ok(amt + n), } } } impl Write for GzEncoder { fn write(&mut self, buf: &[u8]) -> io::Result { self.get_mut().write(buf) } fn flush(&mut self) -> io::Result<()> { self.get_mut().flush() } } /// A gzip streaming decoder /// /// This structure consumes a [`BufRead`] interface, reading compressed data /// from the underlying reader, and emitting uncompressed data. /// /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples /// /// ``` /// use std::io::prelude::*; /// use std::io; /// # use flate2::Compression; /// # use flate2::write::GzEncoder; /// use flate2::bufread::GzDecoder; /// /// # fn main() { /// # let mut e = GzEncoder::new(Vec::new(), Compression::default()); /// # e.write_all(b"Hello World").unwrap(); /// # let bytes = e.finish().unwrap(); /// # println!("{}", decode_reader(bytes).unwrap()); /// # } /// # /// // Uncompresses a Gz Encoded vector of bytes and returns a string or error /// // Here &[u8] implements BufRead /// /// fn decode_reader(bytes: Vec) -> io::Result { /// let mut gz = GzDecoder::new(&bytes[..]); /// let mut s = String::new(); /// gz.read_to_string(&mut s)?; /// Ok(s) /// } /// ``` #[derive(Debug)] pub struct GzDecoder { inner: GzState, header: Option, reader: CrcReader>, multi: bool, } #[derive(Debug)] pub enum GzHeaderParsingState { Start, Xlen, Extra, Filename, Comment, Crc, } #[derive(Debug)] pub struct GzHeaderPartial { buf: Vec, state: GzHeaderParsingState, flg: u8, xlen: u16, crc: Crc, header: GzHeader, } impl GzHeaderPartial { fn new() -> GzHeaderPartial { GzHeaderPartial { buf: Vec::with_capacity(10), // minimum header length state: GzHeaderParsingState::Start, flg: 0, xlen: 0, crc: Crc::new(), header: GzHeader { extra: None, filename: None, comment: None, operating_system: 0, mtime: 0, }, } } pub fn take_header(self) -> GzHeader { self.header } } #[derive(Debug)] enum GzState { Header(GzHeaderPartial), Body, Finished(usize, [u8; 8]), Err(io::Error), End, } /// A small adapter which reads data originally from `buf` and then reads all /// further data from `reader`. This will also buffer all data read from /// `reader` into `buf` for reuse on a further call. struct Buffer<'a, T: 'a> { part: &'a mut GzHeaderPartial, buf_cur: usize, buf_max: usize, reader: &'a mut T, } impl<'a, T> Buffer<'a, T> { fn new(part: &'a mut GzHeaderPartial, reader: &'a mut T) -> Buffer<'a, T> { Buffer { reader, buf_cur: 0, buf_max: part.buf.len(), part, } } } impl<'a, T: Read> Read for Buffer<'a, T> { fn read(&mut self, buf: &mut [u8]) -> io::Result { let mut bufref = match self.part.state { GzHeaderParsingState::Filename => self.part.header.filename.as_mut(), GzHeaderParsingState::Comment => self.part.header.comment.as_mut(), _ => None, }; if let Some(ref mut b) = bufref { // we have a direct reference to a buffer where to write let len = self.reader.read(buf)?; if len > 0 && buf[len - 1] == 0 { // we do not append the final 0 b.extend_from_slice(&buf[..len - 1]); } else { b.extend_from_slice(&buf[..len]); } self.part.crc.update(&buf[..len]); Ok(len) } else if self.buf_cur == self.buf_max { // we read new bytes and also save them in self.part.buf let len = self.reader.read(buf)?; self.part.buf.extend_from_slice(&buf[..len]); self.part.crc.update(&buf[..len]); Ok(len) } else { // we first read the previously saved bytes let len = (&self.part.buf[self.buf_cur..self.buf_max]).read(buf)?; self.buf_cur += len; Ok(len) } } } impl<'a, T> Buffer<'a, T> where T: std::io::Read, { // If we manage to read all the bytes, we reset the buffer fn read_and_forget(&mut self, buf: &mut [u8]) -> io::Result { self.read_exact(buf)?; // we managed to read the whole buf // we will no longer need the previously saved bytes in self.part.buf let rlen = buf.len(); self.part.buf.truncate(0); self.buf_cur = 0; self.buf_max = 0; Ok(rlen) } } impl GzDecoder { /// Creates a new decoder from the given reader, immediately parsing the /// gzip header. pub fn new(mut r: R) -> GzDecoder { let mut part = GzHeaderPartial::new(); let mut header = None; let result = { let mut reader = Buffer::new(&mut part, &mut r); read_gz_header_part(&mut reader) }; let state = match result { Ok(()) => { header = Some(part.take_header()); GzState::Body } Err(ref err) if io::ErrorKind::WouldBlock == err.kind() => GzState::Header(part), Err(err) => GzState::Err(err), }; GzDecoder { inner: state, reader: CrcReader::new(deflate::bufread::DeflateDecoder::new(r)), multi: false, header, } } fn multi(mut self, flag: bool) -> GzDecoder { self.multi = flag; self } } impl GzDecoder { /// Returns the header associated with this stream, if it was valid pub fn header(&self) -> Option<&GzHeader> { self.header.as_ref() } /// Acquires a reference to the underlying reader. pub fn get_ref(&self) -> &R { self.reader.get_ref().get_ref() } /// Acquires a mutable reference to the underlying stream. /// /// Note that mutation of the stream may result in surprising results if /// this encoder is continued to be used. pub fn get_mut(&mut self) -> &mut R { self.reader.get_mut().get_mut() } /// Consumes this decoder, returning the underlying reader. pub fn into_inner(self) -> R { self.reader.into_inner().into_inner() } } impl Read for GzDecoder { fn read(&mut self, into: &mut [u8]) -> io::Result { let GzDecoder { inner, header, reader, multi, } = self; loop { *inner = match mem::replace(inner, GzState::End) { GzState::Header(mut part) => { let result = { let mut reader = Buffer::new(&mut part, reader.get_mut().get_mut()); read_gz_header_part(&mut reader) }; match result { Ok(()) => { *header = Some(part.take_header()); GzState::Body } Err(err) if io::ErrorKind::WouldBlock == err.kind() => { *inner = GzState::Header(part); return Err(err); } Err(err) => return Err(err), } } GzState::Body => { if into.is_empty() { *inner = GzState::Body; return Ok(0); } let n = reader.read(into).map_err(|err| { if io::ErrorKind::WouldBlock == err.kind() { *inner = GzState::Body; } err })?; match n { 0 => GzState::Finished(0, [0; 8]), n => { *inner = GzState::Body; return Ok(n); } } } GzState::Finished(pos, mut buf) => { if pos < buf.len() { let n = reader .get_mut() .get_mut() .read(&mut buf[pos..]) .and_then(|n| { if n == 0 { Err(io::ErrorKind::UnexpectedEof.into()) } else { Ok(n) } }) .map_err(|err| { if io::ErrorKind::WouldBlock == err.kind() { *inner = GzState::Finished(pos, buf); } err })?; GzState::Finished(pos + n, buf) } else { let (crc, amt) = finish(&buf); if crc != reader.crc().sum() || amt != reader.crc().amount() { return Err(corrupt()); } else if *multi { let is_eof = reader .get_mut() .get_mut() .fill_buf() .map(|buf| buf.is_empty()) .map_err(|err| { if io::ErrorKind::WouldBlock == err.kind() { *inner = GzState::Finished(pos, buf); } err })?; if is_eof { GzState::End } else { reader.reset(); reader.get_mut().reset_data(); header.take(); GzState::Header(GzHeaderPartial::new()) } } else { GzState::End } } } GzState::Err(err) => return Err(err), GzState::End => return Ok(0), }; } } } impl Write for GzDecoder { fn write(&mut self, buf: &[u8]) -> io::Result { self.get_mut().write(buf) } fn flush(&mut self) -> io::Result<()> { self.get_mut().flush() } } /// A gzip streaming decoder that decodes all members of a multistream /// /// A gzip member consists of a header, compressed data and a trailer. The [gzip /// specification](https://tools.ietf.org/html/rfc1952), however, allows multiple /// gzip members to be joined in a single stream. `MultiGzDecoder` will /// decode all consecutive members while `GzDecoder` will only decompress /// the first gzip member. The multistream format is commonly used in /// bioinformatics, for example when using the BGZF compressed data. /// /// This structure exposes a [`BufRead`] interface that will consume all gzip members /// from the underlying reader and emit uncompressed data. /// /// [`BufRead`]: https://doc.rust-lang.org/std/io/trait.BufRead.html /// /// # Examples /// /// ``` /// use std::io::prelude::*; /// use std::io; /// # use flate2::Compression; /// # use flate2::write::GzEncoder; /// use flate2::bufread::MultiGzDecoder; /// /// # fn main() { /// # let mut e = GzEncoder::new(Vec::new(), Compression::default()); /// # e.write_all(b"Hello World").unwrap(); /// # let bytes = e.finish().unwrap(); /// # println!("{}", decode_reader(bytes).unwrap()); /// # } /// # /// // Uncompresses a Gz Encoded vector of bytes and returns a string or error /// // Here &[u8] implements BufRead /// /// fn decode_reader(bytes: Vec) -> io::Result { /// let mut gz = MultiGzDecoder::new(&bytes[..]); /// let mut s = String::new(); /// gz.read_to_string(&mut s)?; /// Ok(s) /// } /// ``` #[derive(Debug)] pub struct MultiGzDecoder(GzDecoder); impl MultiGzDecoder { /// Creates a new decoder from the given reader, immediately parsing the /// (first) gzip header. If the gzip stream contains multiple members all will /// be decoded. pub fn new(r: R) -> MultiGzDecoder { MultiGzDecoder(GzDecoder::new(r).multi(true)) } } impl MultiGzDecoder { /// Returns the current header associated with this stream, if it's valid pub fn header(&self) -> Option<&GzHeader> { self.0.header() } /// Acquires a reference to the underlying reader. pub fn get_ref(&self) -> &R { self.0.get_ref() } /// Acquires a mutable reference to the underlying stream. /// /// Note that mutation of the stream may result in surprising results if /// this encoder is continued to be used. pub fn get_mut(&mut self) -> &mut R { self.0.get_mut() } /// Consumes this decoder, returning the underlying reader. pub fn into_inner(self) -> R { self.0.into_inner() } } impl Read for MultiGzDecoder { fn read(&mut self, into: &mut [u8]) -> io::Result { self.0.read(into) } } #[cfg(test)] pub mod tests { use crate::gz::bufread::*; use std::io; use std::io::{Cursor, Read, Write}; //a cursor turning EOF into blocking errors #[derive(Debug)] pub struct BlockingCursor { pub cursor: Cursor>, } impl BlockingCursor { pub fn new() -> BlockingCursor { BlockingCursor { cursor: Cursor::new(Vec::new()), } } pub fn set_position(&mut self, pos: u64) { return self.cursor.set_position(pos); } pub fn position(&mut self) -> u64 { return self.cursor.position(); } } impl Write for BlockingCursor { fn write(&mut self, buf: &[u8]) -> io::Result { return self.cursor.write(buf); } fn flush(&mut self) -> io::Result<()> { return self.cursor.flush(); } } impl Read for BlockingCursor { fn read(&mut self, buf: &mut [u8]) -> io::Result { //use the cursor, except it turns eof into blocking error let r = self.cursor.read(buf); match r { Err(ref err) => { if err.kind() == io::ErrorKind::UnexpectedEof { return Err(io::ErrorKind::WouldBlock.into()); } } Ok(0) => { //regular EOF turned into blocking error return Err(io::ErrorKind::WouldBlock.into()); } Ok(_n) => {} } return r; } } #[test] // test function read_and_forget of Buffer fn buffer_read_and_forget() { // this is unused except for the buffering let mut part = GzHeaderPartial::new(); // this is a reader which receives data afterwards let mut r = BlockingCursor::new(); let data = vec![1, 2, 3]; let mut out = Vec::with_capacity(7); match r.write_all(&data) { Ok(()) => {} _ => { panic!("Unexpected result for write_all"); } } r.set_position(0); // First read : successful for one byte let mut reader = Buffer::new(&mut part, &mut r); out.resize(1, 0); match reader.read_and_forget(&mut out) { Ok(1) => {} _ => { panic!("Unexpected result for read_and_forget with data"); } } // Second read : incomplete for 7 bytes (we have only 2) out.resize(7, 0); match reader.read_and_forget(&mut out) { Err(ref err) => { assert_eq!(io::ErrorKind::WouldBlock, err.kind()); } _ => { panic!("Unexpected result for read_and_forget with incomplete"); } } // 3 more data bytes have arrived let pos = r.position(); let data2 = vec![4, 5, 6]; match r.write_all(&data2) { Ok(()) => {} _ => { panic!("Unexpected result for write_all"); } } r.set_position(pos); // Third read : still incomplete for 7 bytes (we have 5) let mut reader2 = Buffer::new(&mut part, &mut r); match reader2.read_and_forget(&mut out) { Err(ref err) => { assert_eq!(io::ErrorKind::WouldBlock, err.kind()); } _ => { panic!("Unexpected result for read_and_forget with more incomplete"); } } // 3 more data bytes have arrived again let pos2 = r.position(); let data3 = vec![7, 8, 9]; match r.write_all(&data3) { Ok(()) => {} _ => { panic!("Unexpected result for write_all"); } } r.set_position(pos2); // Fourth read : now successful for 7 bytes let mut reader3 = Buffer::new(&mut part, &mut r); match reader3.read_and_forget(&mut out) { Ok(7) => { assert_eq!(out[0], 2); assert_eq!(out[6], 8); } _ => { panic!("Unexpected result for read_and_forget with data"); } } // Fifth read : successful for one more byte out.resize(1, 0); match reader3.read_and_forget(&mut out) { Ok(1) => { assert_eq!(out[0], 9); } _ => { panic!("Unexpected result for read_and_forget with data"); } } } }