diff options
Diffstat (limited to 'lzip.h')
| -rw-r--r-- | lzip.h | 523 |
1 files changed, 523 insertions, 0 deletions
@@ -0,0 +1,523 @@ +/* Lziprecover - Data recovery tool for the lzip format + Copyright (C) 2009-2022 Antonio Diaz Diaz. + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 2 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see <http://www.gnu.org/licenses/>. +*/ + +#include "common.h" + +class State + { + int st; + +public: + enum { states = 12 }; + State() : st( 0 ) {} + int operator()() const { return st; } + bool is_char() const { return st < 7; } + + void set_char() + { + static const int next[states] = { 0, 0, 0, 0, 1, 2, 3, 4, 5, 6, 4, 5 }; + st = next[st]; + } + bool is_char_set_char() { set_char(); return st < 4; } + void set_match() { st = ( st < 7 ) ? 7 : 10; } + void set_rep() { st = ( st < 7 ) ? 8 : 11; } + void set_short_rep() { st = ( st < 7 ) ? 9 : 11; } + }; + + +enum { + min_dictionary_bits = 12, + min_dictionary_size = 1 << min_dictionary_bits, // >= modeled_distances + max_dictionary_bits = 29, + max_dictionary_size = 1 << max_dictionary_bits, + min_member_size = 36, + literal_context_bits = 3, + literal_pos_state_bits = 0, // not used + pos_state_bits = 2, + pos_states = 1 << pos_state_bits, + pos_state_mask = pos_states - 1, + + len_states = 4, + dis_slot_bits = 6, + start_dis_model = 4, + end_dis_model = 14, + modeled_distances = 1 << (end_dis_model / 2), // 128 + dis_align_bits = 4, + dis_align_size = 1 << dis_align_bits, + + len_low_bits = 3, + len_mid_bits = 3, + len_high_bits = 8, + len_low_symbols = 1 << len_low_bits, + len_mid_symbols = 1 << len_mid_bits, + len_high_symbols = 1 << len_high_bits, + max_len_symbols = len_low_symbols + len_mid_symbols + len_high_symbols, + + min_match_len = 2, // must be 2 + max_match_len = min_match_len + max_len_symbols - 1, // 273 + min_match_len_limit = 5 }; + +inline int get_len_state( const int len ) + { return std::min( len - min_match_len, len_states - 1 ); } + +inline int get_lit_state( const uint8_t prev_byte ) + { return prev_byte >> ( 8 - literal_context_bits ); } + + +enum { bit_model_move_bits = 5, + bit_model_total_bits = 11, + bit_model_total = 1 << bit_model_total_bits }; + +struct Bit_model + { + int probability; + Bit_model() : probability( bit_model_total / 2 ) {} + }; + +struct Len_model + { + Bit_model choice1; + Bit_model choice2; + Bit_model bm_low[pos_states][len_low_symbols]; + Bit_model bm_mid[pos_states][len_mid_symbols]; + Bit_model bm_high[len_high_symbols]; + }; + + +// defined in main.cc +extern int verbosity; + +class Pretty_print // requires global var 'int verbosity' + { + std::string name_; + std::string padded_name; + const char * const stdin_name; + unsigned longest_name; + mutable bool first_post; + +public: + Pretty_print( const std::vector< std::string > & filenames ) + : stdin_name( "(stdin)" ), longest_name( 0 ), first_post( false ) + { + if( verbosity <= 0 ) return; + const unsigned stdin_name_len = std::strlen( stdin_name ); + for( unsigned i = 0; i < filenames.size(); ++i ) + { + const std::string & s = filenames[i]; + const unsigned len = ( s == "-" ) ? stdin_name_len : s.size(); + if( longest_name < len ) longest_name = len; + } + if( longest_name == 0 ) longest_name = stdin_name_len; + } + + Pretty_print( const std::string & filename ) + : stdin_name( "(stdin)" ), first_post( false ) + { + const unsigned stdin_name_len = std::strlen( stdin_name ); + longest_name = ( filename == "-" ) ? stdin_name_len : filename.size(); + if( longest_name == 0 ) longest_name = stdin_name_len; + set_name( filename ); + } + + void set_name( const std::string & filename ) + { + if( filename.size() && filename != "-" ) name_ = filename; + else name_ = stdin_name; + padded_name = " "; padded_name += name_; padded_name += ": "; + if( longest_name > name_.size() ) + padded_name.append( longest_name - name_.size(), ' ' ); + first_post = true; + } + + void reset() const { if( name_.size() ) first_post = true; } + const char * name() const { return name_.c_str(); } + void operator()( const char * const msg = 0, FILE * const f = stderr ) const; + }; + + +class CRC32 + { + uint32_t data[256]; // Table of CRCs of all 8-bit messages. + +public: + CRC32() + { + for( unsigned n = 0; n < 256; ++n ) + { + unsigned c = n; + for( int k = 0; k < 8; ++k ) + { if( c & 1 ) c = 0xEDB88320U ^ ( c >> 1 ); else c >>= 1; } + data[n] = c; + } + } + + uint32_t operator[]( const uint8_t byte ) const { return data[byte]; } + + void update_byte( uint32_t & crc, const uint8_t byte ) const + { crc = data[(crc^byte)&0xFF] ^ ( crc >> 8 ); } + + // about as fast as it is possible without messing with endianness + void update_buf( uint32_t & crc, const uint8_t * const buffer, + const int size ) const + { + uint32_t c = crc; + for( int i = 0; i < size; ++i ) + c = data[(c^buffer[i])&0xFF] ^ ( c >> 8 ); + crc = c; + } + + uint32_t compute_crc( const uint8_t * const buffer, + const long long size ) const + { + uint32_t crc = 0xFFFFFFFFU; + for( long long i = 0; i < size; ++i ) + crc = data[(crc^buffer[i])&0xFF] ^ ( crc >> 8 ); + return crc ^ 0xFFFFFFFFU; + } + }; + +extern const CRC32 crc32; + + +inline bool isvalid_ds( const unsigned dictionary_size ) + { return ( dictionary_size >= min_dictionary_size && + dictionary_size <= max_dictionary_size ); } + + +inline int real_bits( unsigned value ) + { + int bits = 0; + while( value > 0 ) { value >>= 1; ++bits; } + return bits; + } + + +const uint8_t lzip_magic[4] = { 0x4C, 0x5A, 0x49, 0x50 }; // "LZIP" + +struct Lzip_header + { + uint8_t data[6]; // 0-3 magic bytes + // 4 version + // 5 coded dictionary size + enum { size = 6 }; + + void set_magic() { std::memcpy( data, lzip_magic, 4 ); data[4] = 1; } + bool verify_magic() const + { return ( std::memcmp( data, lzip_magic, 4 ) == 0 ); } + + bool verify_prefix( const int sz ) const // detect (truncated) header + { + for( int i = 0; i < sz && i < 4; ++i ) + if( data[i] != lzip_magic[i] ) return false; + return ( sz > 0 ); + } + bool verify_corrupt() const // detect corrupt header + { + int matches = 0; + for( int i = 0; i < 4; ++i ) + if( data[i] == lzip_magic[i] ) ++matches; + return ( matches > 1 && matches < 4 ); + } + + uint8_t version() const { return data[4]; } + bool verify_version() const { return ( data[4] == 1 ); } + + unsigned dictionary_size() const + { + unsigned sz = ( 1 << ( data[5] & 0x1F ) ); + if( sz > min_dictionary_size ) + sz -= ( sz / 16 ) * ( ( data[5] >> 5 ) & 7 ); + return sz; + } + + bool dictionary_size( const unsigned sz ) + { + if( !isvalid_ds( sz ) ) return false; + data[5] = real_bits( sz - 1 ); + if( sz > min_dictionary_size ) + { + const unsigned base_size = 1 << data[5]; + const unsigned fraction = base_size / 16; + for( unsigned i = 7; i >= 1; --i ) + if( base_size - ( i * fraction ) >= sz ) + { data[5] |= ( i << 5 ); break; } + } + return true; + } + + bool verify( const bool ignore_bad_ds ) const + { return verify_magic() && verify_version() && + ( ignore_bad_ds || isvalid_ds( dictionary_size() ) ); } + }; + + +struct Lzip_trailer + { + uint8_t data[20]; // 0-3 CRC32 of the uncompressed data + // 4-11 size of the uncompressed data + // 12-19 member size including header and trailer + enum { size = 20 }; + + unsigned data_crc() const + { + unsigned tmp = 0; + for( int i = 3; i >= 0; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } + + void data_crc( unsigned crc ) + { for( int i = 0; i <= 3; ++i ) { data[i] = (uint8_t)crc; crc >>= 8; } } + + unsigned long long data_size() const + { + unsigned long long tmp = 0; + for( int i = 11; i >= 4; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } + + void data_size( unsigned long long sz ) + { for( int i = 4; i <= 11; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } } + + unsigned long long member_size() const + { + unsigned long long tmp = 0; + for( int i = 19; i >= 12; --i ) { tmp <<= 8; tmp += data[i]; } + return tmp; + } + + void member_size( unsigned long long sz ) + { for( int i = 12; i <= 19; ++i ) { data[i] = (uint8_t)sz; sz >>= 8; } } + + bool verify_consistency() const // check internal consistency + { + const unsigned crc = data_crc(); + const unsigned long long dsize = data_size(); + if( ( crc == 0 ) != ( dsize == 0 ) ) return false; + const unsigned long long msize = member_size(); + if( msize < min_member_size ) return false; + const unsigned long long mlimit = ( 9 * dsize + 7 ) / 8 + min_member_size; + if( mlimit > dsize && msize > mlimit ) return false; + const unsigned long long dlimit = 7090 * ( msize - 26 ) - 1; + if( dlimit > msize && dsize > dlimit ) return false; + return true; + } + }; + + +#ifndef INT64_MAX +#define INT64_MAX 0x7FFFFFFFFFFFFFFFLL +#endif + +class Block + { + long long pos_, size_; // pos + size <= INT64_MAX + +public: + Block( const long long p, const long long s ) : pos_( p ), size_( s ) {} + + long long pos() const { return pos_; } + long long size() const { return size_; } + long long end() const { return pos_ + size_; } + + void pos( const long long p ) { pos_ = p; } + void size( const long long s ) { size_ = s; } + + bool operator==( const Block & b ) const + { return pos_ == b.pos_ && size_ == b.size_; } + bool operator!=( const Block & b ) const + { return pos_ != b.pos_ || size_ != b.size_; } + + bool operator<( const Block & b ) const { return pos_ < b.pos_; } + + bool includes( const long long pos ) const + { return ( pos_ <= pos && end() > pos ); } + bool overlaps( const Block & b ) const + { return ( pos_ < b.end() && b.pos_ < end() ); } + bool overlaps( const long long pos, const long long size ) const + { return ( pos_ < pos + size && pos < end() ); } + + void shift( Block & b ) { ++size_; ++b.pos_; --b.size_; } + Block split( const long long pos ); + }; + + +struct Member_list // members/gaps/tdata to be dumped/removed/stripped + { + bool damaged; + bool tdata; + bool in, rin; + std::vector< Block > range_vector, rrange_vector; + + Member_list() : damaged( false ), tdata( false ), in( true ), rin( true ) {} + void parse_ml( const char * p, const char * const option_name ); + + bool range() const { return range_vector.size() || rrange_vector.size(); } + + // blocks is the sum of members + gaps, excluding trailing data + bool includes( const long i, const long blocks ) const + { + for( unsigned j = 0; j < range_vector.size(); ++j ) + { + if( range_vector[j].pos() > i ) break; + if( range_vector[j].end() > i ) return in; + } + if( i >= 0 && i < blocks ) + for( unsigned j = 0; j < rrange_vector.size(); ++j ) + { + if( rrange_vector[j].pos() > blocks - i - 1 ) break; + if( rrange_vector[j].end() > blocks - i - 1 ) return rin; + } + return !in || !rin; + } + }; + + +struct Error + { + const char * const msg; + explicit Error( const char * const s ) : msg( s ) {} + }; + +inline unsigned long long positive_diff( const unsigned long long x, + const unsigned long long y ) + { return ( ( x > y ) ? x - y : 0 ); } + +inline void set_retval( int & retval, const int new_val ) + { if( retval < new_val ) retval = new_val; } + +const char * const bad_magic_msg = "Bad magic number (file not in lzip format)."; +const char * const bad_dict_msg = "Invalid dictionary size in member header."; +const char * const corrupt_mm_msg = "Corrupt header in multimember file."; +const char * const trailing_msg = "Trailing data not allowed."; + +// defined in alone_to_lz.cc +int alone_to_lz( const int infd, const Pretty_print & pp ); + +// defined in decoder.cc +long long readblock( const int fd, uint8_t * const buf, const long long size ); +long long writeblock( const int fd, const uint8_t * const buf, + const long long size ); + +// defined in dump_remove.cc +int dump_members( const std::vector< std::string > & filenames, + const std::string & default_output_filename, + const Member_list & member_list, const bool force, + bool ignore_errors, bool ignore_trailing, + const bool loose_trailing, const bool strip, + const bool to_stdout ); +int remove_members( const std::vector< std::string > & filenames, + const Member_list & member_list, bool ignore_errors, + bool ignore_trailing, const bool loose_trailing ); + +// defined in list.cc +int list_files( const std::vector< std::string > & filenames, + const bool ignore_errors, + const bool ignore_trailing, const bool loose_trailing ); + +// defined in lzip_index.cc +int seek_read( const int fd, uint8_t * const buf, const int size, + const long long pos ); + +// defined in lunzcrash.cc +int lunzcrash_bit( const char * const input_filename ); +int lunzcrash_block( const char * const input_filename, const int sector_size ); +int md5sum_files( const std::vector< std::string > & filenames ); + +// defined in main.cc +extern const char * const program_name; +extern std::string output_filename; // global vars for output file +extern int outfd; +struct stat; +const char * bad_version( const unsigned version ); +const char * format_ds( const unsigned dictionary_size ); +void show_header( const unsigned dictionary_size ); +int open_instream( const char * const name, struct stat * const in_statsp, + const bool one_to_one, const bool reg_only = false ); +int open_truncable_stream( const char * const name, + struct stat * const in_statsp ); +bool open_outstream( const bool force, const bool protect, + const bool rw = false, const bool skipping = true ); +bool file_exists( const std::string & filename ); +void cleanup_and_fail( const int retval ); +bool check_tty_out(); +void set_signal_handler(); +int close_outstream( const struct stat * const in_statsp ); +std::string insert_fixed( std::string name ); +void show_2file_error( const char * const msg1, const char * const name1, + const char * const name2, const char * const msg2 ); +class Range_decoder; +void show_dprogress( const unsigned long long cfile_size = 0, + const unsigned long long partial_size = 0, + const Range_decoder * const d = 0, + const Pretty_print * const p = 0 ); + +// defined in merge.cc +bool copy_file( const int infd, const int outfd, + const long long max_size = -1 ); +int test_member_from_file( const int infd, const unsigned long long msize, + long long * const failure_posp = 0 ); +int merge_files( const std::vector< std::string > & filenames, + const std::string & default_output_filename, + const char terminator, const bool force ); + +// defined in nrep_stats.cc +int print_nrep_stats( const std::vector< std::string > & filenames, + const int repeated_byte, const bool ignore_errors, + const bool ignore_trailing, const bool loose_trailing ); + +// defined in range_dec.cc +const char * format_num( unsigned long long num, + unsigned long long limit = -1ULL, + const int set_prefix = 0 ); +bool safe_seek( const int fd, const long long pos ); +int range_decompress( const std::string & input_filename, + const std::string & default_output_filename, + Block range, const bool force, const bool ignore_errors, + const bool ignore_trailing, const bool loose_trailing, + const bool to_stdout ); + +// defined in repair.cc +long long seek_write( const int fd, const uint8_t * const buf, + const long long size, const long long pos ); +uint8_t * read_member( const int infd, const long long mpos, + const long long msize ); +int repair_file( const std::string & input_filename, + const std::string & default_output_filename, + const char terminator, const bool force ); +int debug_delay( const std::string & input_filename, Block range, + const char terminator ); +int debug_repair( const std::string & input_filename, + const Bad_byte & bad_byte, const char terminator ); +int debug_decompress( const std::string & input_filename, + const Bad_byte & bad_byte, const bool show_packets ); + +// defined in reproduce.cc +int reproduce_file( const std::string & input_filename, + const std::string & default_output_filename, + const char * const lzip_name, + const char * const reference_filename, + const int lzip_level, const char terminator, + const bool force ); +int debug_reproduce_file( const std::string & input_filename, + const char * const lzip_name, + const char * const reference_filename, + const Block & range, const int sector_size, + const int lzip_level ); + +// defined in split.cc +int split_file( const std::string & input_filename, + const std::string & default_output_filename, const bool force ); |