diff options
Diffstat (limited to 'merge.cc')
| -rw-r--r-- | merge.cc | 185 |
1 files changed, 107 insertions, 78 deletions
@@ -1,5 +1,5 @@ /* Lziprecover - Data recovery tool for the lzip format - Copyright (C) 2009-2018 Antonio Diaz Diaz. + Copyright (C) 2009-2019 Antonio Diaz Diaz. This program is free software: you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -31,19 +31,30 @@ #include "lzip.h" #include "decoder.h" -#include "block.h" -#include "file_index.h" +#include "lzip_index.h" +Block Block::split( const long long pos ) + { + if( pos > pos_ && pos < end() ) + { + const Block b( pos_, pos - pos_ ); + pos_ = pos; size_ -= b.size_; + return b; + } + return Block( 0, 0 ); + } + namespace { bool pending_newline = false; -void print_pending_newline() - { if( pending_newline ) std::fputc( '\n', stdout ); pending_newline = false; } +void print_pending_newline( const char terminator ) + { if( pending_newline && terminator != '\n' ) std::fputc( '\n', stdout ); + pending_newline = false; } -bool file_crc( uint32_t & crc, const int infd ) +bool file_crc( uint32_t & crc, const int infd, const char * const filename ) { const int buffer_size = 65536; crc = 0xFFFFFFFFU; @@ -54,7 +65,8 @@ bool file_crc( uint32_t & crc, const int infd ) { const int rd = readblock( infd, buffer, buffer_size ); if( rd != buffer_size && errno ) - { show_error( "Error reading input file", errno ); error = true; break; } + { show_file_error( filename, "Error reading input file", errno ); + error = true; break; } if( rd > 0 ) crc32.update_buf( crc, buffer, rd ); if( rd < buffer_size ) break; // EOF @@ -108,6 +120,7 @@ void combine( std::vector< Block > & block_vector, std::vector< Block > & bv ) // positions in 'block_vector' are absolute file positions. // blocks in 'block_vector' are ascending and don't overlap. bool diff_member( const long long mpos, const long long msize, + const std::vector< std::string > & filenames, const std::vector< int > & infd_vector, std::vector< Block > & block_vector, std::vector< int > & color_vector ) @@ -138,12 +151,13 @@ bool diff_member( const long long mpos, const long long msize, const int size = std::min( (long long)buffer_size, msize - partial_pos ); const int rd = readblock( fd1, buffer1, size ); if( rd != size && errno ) - { show_error( "Error reading input file", errno ); error = true; break; } + { show_file_error( filenames[i1].c_str(), "Error reading input file", + errno ); error = true; break; } if( rd > 0 ) { if( readblock( fd2, buffer2, rd ) != rd ) - { show_error( "Error reading input file", errno ); - error = true; break; } + { show_file_error( filenames[i2].c_str(), "Error reading input file", + errno ); error = true; break; } for( int i = 0; i < rd; ++i ) { if( buffer1[i] != buffer2[i] ) @@ -201,13 +215,13 @@ long ipow( const unsigned base, const unsigned exponent ) int open_input_files( const std::vector< std::string > & filenames, std::vector< int > & infd_vector, - File_index & file_index, struct stat * const in_statsp ) + Lzip_index & lzip_index, struct stat * const in_statsp ) { const int files = filenames.size(); for( int i = 0; i + 1 < files; ++i ) for( int j = i + 1; j < files; ++j ) if( filenames[i] == filenames[j] ) - { show_error2( "Input file", filenames[i].c_str(), "given twice." ); + { show_file_error( filenames[i].c_str(), "Input file given twice." ); return 2; } { std::vector< uint32_t > crc_vector( files ); @@ -217,51 +231,52 @@ int open_input_files( const std::vector< std::string > & filenames, infd_vector[i] = open_instream( filenames[i].c_str(), ( i == 0 ) ? in_statsp : &in_stats, true, true ); if( infd_vector[i] < 0 ) return 1; - if( !file_crc( crc_vector[i], infd_vector[i] ) ) return 1; + if( !file_crc( crc_vector[i], infd_vector[i], filenames[i].c_str() ) ) + return 1; for( int j = 0; j < i; ++j ) if( crc_vector[i] == crc_vector[j] ) - { show_error4( "Input files", filenames[j].c_str(), - filenames[i].c_str(), "are identical." ); return 2; } + { show_2file_error( "Input files", filenames[j].c_str(), + filenames[i].c_str(), "are identical." ); return 2; } } } - long long isize = 0; - int good_fi = -1; + long long insize = 0; + int good_i = -1; for( int i = 0; i < files; ++i ) { long long tmp; - const File_index fi( infd_vector[i], true, true, true ); - if( fi.retval() == 0 ) // file format is intact + const Lzip_index li( infd_vector[i], true, true, true ); + if( li.retval() == 0 ) // file format is intact { - if( good_fi < 0 ) { good_fi = i; file_index = fi; } - else if( file_index != fi ) - { show_error4( "Input files", filenames[good_fi].c_str(), - filenames[i].c_str(), "are different." ); return 2; } - tmp = file_index.file_size(); + if( good_i < 0 ) { good_i = i; lzip_index = li; } + else if( lzip_index != li ) + { show_2file_error( "Input files", filenames[good_i].c_str(), + filenames[i].c_str(), "are different." ); return 2; } + tmp = lzip_index.file_size(); } else // file format is damaged { tmp = lseek( infd_vector[i], 0, SEEK_END ); if( tmp < 0 ) { - show_error2( "Input file", filenames[i].c_str(), "is not seekable." ); + show_file_error( filenames[i].c_str(), "Input file is not seekable." ); return 1; } } if( tmp < min_member_size ) - { show_error2( "Input file", filenames[i].c_str(), "is too short." ); + { show_file_error( filenames[i].c_str(), "Input file is too short." ); return 2; } - if( i == 0 ) isize = tmp; - else if( isize != tmp ) - { show_error4( "Sizes of input files", filenames[0].c_str(), - filenames[i].c_str(), "are different." ); return 2; } + if( i == 0 ) insize = tmp; + else if( insize != tmp ) + { show_2file_error( "Sizes of input files", filenames[0].c_str(), + filenames[i].c_str(), "are different." ); return 2; } } - if( file_index.retval() != 0 ) + if( lzip_index.retval() != 0 ) { - const File_index fi( infd_vector, isize ); - if( fi.retval() == 0 ) // file format could be recovered - file_index = fi; + const Lzip_index li( infd_vector, insize ); + if( li.retval() == 0 ) // file format could be recovered + lzip_index = li; else { show_error( "Format damaged in all input files." ); return 2; } } @@ -270,12 +285,12 @@ int open_input_files( const std::vector< std::string > & filenames, { const int infd = infd_vector[i]; bool error = false; - for( long j = 0; j < file_index.members(); ++j ) + for( long j = 0; j < lzip_index.members(); ++j ) { - const long long mpos = file_index.mblock( j ).pos(); - const long long msize = file_index.mblock( j ).size(); + const long long mpos = lzip_index.mblock( j ).pos(); + const long long msize = lzip_index.mblock( j ).size(); if( !safe_seek( infd, mpos ) ) return 1; - if( !test_member_from_file( infd, msize ) ) { error = true; break; } + if( test_member_from_file( infd, msize ) != 0 ) { error = true; break; } } if( !error ) { @@ -333,7 +348,8 @@ bool color_done( const std::vector< int > & color_vector, const int i ) bool try_merge_member2( const long long mpos, const long long msize, const std::vector< Block > & block_vector, const std::vector< int > & color_vector, - const std::vector< int > & infd_vector ) + const std::vector< int > & infd_vector, + const char terminator ) { const int blocks = block_vector.size(); const int files = infd_vector.size(); @@ -355,8 +371,8 @@ bool try_merge_member2( const long long mpos, const long long msize, { if( verbosity >= 2 ) { - std::printf( " Trying variation %d of %d, block %d \r", - var, variations, bi + 1 ); + std::printf( " Trying variation %d of %d, block %d %c", + var, variations, bi + 1, terminator ); std::fflush( stdout ); pending_newline = true; } if( !safe_seek( infd, block_vector[bi].pos() ) || @@ -365,7 +381,8 @@ bool try_merge_member2( const long long mpos, const long long msize, !safe_seek( outfd, mpos ) ) cleanup_and_fail( 1 ); long long failure_pos = 0; - if( test_member_from_file( outfd, msize, &failure_pos ) ) return true; + if( test_member_from_file( outfd, msize, &failure_pos ) == 0 ) + return true; if( mpos + failure_pos < block_vector[bi].end() ) break; } } @@ -377,7 +394,8 @@ bool try_merge_member2( const long long mpos, const long long msize, bool try_merge_member( const long long mpos, const long long msize, const std::vector< Block > & block_vector, const std::vector< int > & color_vector, - const std::vector< int > & infd_vector ) + const std::vector< int > & infd_vector, + const char terminator ) { const int blocks = block_vector.size(); const int files = infd_vector.size(); @@ -400,7 +418,8 @@ bool try_merge_member( const long long mpos, const long long msize, long var = 0; for( int i = 0; i < blocks; ++i ) var = ( var * files ) + file_idx[i]; - std::printf( " Trying variation %ld of %ld \r", var + 1, variations ); + std::printf( " Trying variation %ld of %ld %c", + var + 1, variations, terminator ); std::fflush( stdout ); pending_newline = true; } while( bi < blocks ) @@ -412,10 +431,9 @@ bool try_merge_member( const long long mpos, const long long msize, cleanup_and_fail( 1 ); ++bi; } - if( !safe_seek( outfd, mpos ) ) - cleanup_and_fail( 1 ); + if( !safe_seek( outfd, mpos ) ) cleanup_and_fail( 1 ); long long failure_pos = 0; - if( test_member_from_file( outfd, msize, &failure_pos ) ) return true; + if( test_member_from_file( outfd, msize, &failure_pos ) == 0 ) return true; while( bi > 0 && mpos + failure_pos < block_vector[bi-1].pos() ) --bi; while( --bi >= 0 ) { @@ -433,7 +451,8 @@ bool try_merge_member( const long long mpos, const long long msize, bool try_merge_member1( const long long mpos, const long long msize, const std::vector< Block > & block_vector, const std::vector< int > & color_vector, - const std::vector< int > & infd_vector ) + const std::vector< int > & infd_vector, + const char terminator ) { if( block_vector.size() != 1 || block_vector[0].size() <= 1 ) return false; const long long pos = block_vector[0].pos(); @@ -458,8 +477,8 @@ bool try_merge_member1( const long long mpos, const long long msize, { if( verbosity >= 2 ) { - std::printf( " Trying variation %d of %d, position %lld \r", - var, variations, pos + i ); + std::printf( " Trying variation %d of %d, position %lld %c", + var, variations, pos + i, terminator ); std::fflush( stdout ); pending_newline = true; } if( !safe_seek( outfd, pos + i ) || @@ -468,7 +487,8 @@ bool try_merge_member1( const long long mpos, const long long msize, !safe_seek( outfd, mpos ) ) cleanup_and_fail( 1 ); long long failure_pos = 0; - if( test_member_from_file( outfd, msize, &failure_pos ) ) return true; + if( test_member_from_file( outfd, msize, &failure_pos ) == 0 ) + return true; if( mpos + failure_pos <= pos + i ) break; } } @@ -478,12 +498,15 @@ bool try_merge_member1( const long long mpos, const long long msize, } // end namespace +// infd and outfd can refer to the same file if copying to a lower file +// position or if source and destination blocks don't overlap. // max_size < 0 means no size limit. bool copy_file( const int infd, const int outfd, const long long max_size ) { const int buffer_size = 65536; // remaining number of bytes to copy long long rest = ( ( max_size >= 0 ) ? max_size : buffer_size ); + long long copied_size = 0; uint8_t * const buffer = new uint8_t[buffer_size]; bool error = false; @@ -500,72 +523,78 @@ bool copy_file( const int infd, const int outfd, const long long max_size ) if( wr != rd ) { show_error( "Error writing output file", errno ); error = true; break; } + copied_size += rd; } if( rd < size ) break; // EOF } delete[] buffer; + if( !error && max_size >= 0 && copied_size != max_size ) + { show_error( "Input file ends unexpectedly." ); error = true; } return !error; } -bool test_member_from_file( const int infd, const unsigned long long msize, - long long * const failure_posp ) +// Return value: 0 = OK, 1 = bad msize, 2 = data error +// 'failure_pos' is relative to the beginning of the member +int test_member_from_file( const int infd, const unsigned long long msize, + long long * const failure_posp ) { Range_decoder rdec( infd ); - File_header header; - rdec.read_data( header.data, File_header::size ); + Lzip_header header; + rdec.read_data( header.data, Lzip_header::size ); const unsigned dictionary_size = header.dictionary_size(); + bool done = false; if( !rdec.finished() && header.verify_magic() && header.verify_version() && isvalid_ds( dictionary_size ) ) { LZ_decoder decoder( rdec, dictionary_size, -1 ); const int old_verbosity = verbosity; verbosity = -1; // suppress all messages - Pretty_print dummy( "" ); - const bool done = ( decoder.decode_member( dummy ) == 0 && - rdec.member_position() == msize ); + Pretty_print dummy_pp( "" ); + done = ( decoder.decode_member( dummy_pp ) == 0 ); verbosity = old_verbosity; // restore verbosity level - if( done ) return true; + if( done && rdec.member_position() == msize ) return 0; } if( failure_posp ) *failure_posp = rdec.member_position(); - return false; + return done ? 1 : 2; } int merge_files( const std::vector< std::string > & filenames, const std::string & default_output_filename, - const bool force ) + const bool force, const char terminator ) { const int files = filenames.size(); std::vector< int > infd_vector( files ); - File_index file_index; + Lzip_index lzip_index; struct stat in_stats; const int retval = - open_input_files( filenames, infd_vector, file_index, &in_stats ); + open_input_files( filenames, infd_vector, lzip_index, &in_stats ); if( retval >= 0 ) return retval; if( !safe_seek( infd_vector[0], 0 ) ) return 1; output_filename = default_output_filename.empty() ? insert_fixed( filenames[0] ) : default_output_filename; + set_signal_handler(); if( !open_outstream( force, false, true, false ) ) return 1; if( !copy_file( infd_vector[0], outfd ) ) // copy whole file cleanup_and_fail( 1 ); - for( long j = 0; j < file_index.members(); ++j ) + for( long j = 0; j < lzip_index.members(); ++j ) { - const long long mpos = file_index.mblock( j ).pos(); - const long long msize = file_index.mblock( j ).size(); + const long long mpos = lzip_index.mblock( j ).pos(); + const long long msize = lzip_index.mblock( j ).size(); // vector of data blocks differing among the copies of the current member std::vector< Block > block_vector; // different color means members are different std::vector< int > color_vector( files, 0 ); - if( !diff_member( mpos, msize, infd_vector, block_vector, color_vector ) || - !safe_seek( outfd, mpos ) ) + if( !diff_member( mpos, msize, filenames, infd_vector, block_vector, + color_vector ) || !safe_seek( outfd, mpos ) ) cleanup_and_fail( 1 ); if( block_vector.empty() ) { - if( file_index.members() > 1 && test_member_from_file( outfd, msize ) ) + if( lzip_index.members() > 1 && test_member_from_file( outfd, msize ) == 0 ) continue; if( verbosity >= 0 ) std::fprintf( stderr, "Member %ld is damaged and identical in all files." @@ -576,33 +605,33 @@ int merge_files( const std::vector< std::string > & filenames, if( verbosity >= 2 ) { std::printf( "Merging member %ld of %ld (%lu error%s)\n", - j + 1, file_index.members(), (long)block_vector.size(), + j + 1, lzip_index.members(), (long)block_vector.size(), ( block_vector.size() == 1 ) ? "" : "s" ); std::fflush( stdout ); } bool done = false; - if( file_index.members() > 1 || block_vector.size() > 1 ) + if( lzip_index.members() > 1 || block_vector.size() > 1 ) { if( block_vector.size() > 1 ) { maybe_cluster_blocks( block_vector ); done = try_merge_member2( mpos, msize, block_vector, color_vector, - infd_vector ); - print_pending_newline(); + infd_vector, terminator ); + print_pending_newline( terminator ); } if( !done ) { done = try_merge_member( mpos, msize, block_vector, color_vector, - infd_vector ); - print_pending_newline(); + infd_vector, terminator ); + print_pending_newline( terminator ); } } if( !done ) { done = try_merge_member1( mpos, msize, block_vector, color_vector, - infd_vector ); - print_pending_newline(); + infd_vector, terminator ); + print_pending_newline( terminator ); } if( !done ) { @@ -610,7 +639,7 @@ int merge_files( const std::vector< std::string > & filenames, for( unsigned i = 0; i < block_vector.size(); ++i ) std::fprintf( stderr, "area %2d from position %6lld to %6lld\n", i + 1, block_vector[i].pos(), block_vector[i].end() - 1 ); - show_error( "Some error areas overlap. Can't recover input file." ); + show_error( "Some error areas overlap. Merging is not possible." ); cleanup_and_fail( 2 ); } } |