diff options
Diffstat (limited to 'create.cc')
| -rw-r--r-- | create.cc | 436 |
1 files changed, 225 insertions, 211 deletions
@@ -43,18 +43,53 @@ const CRC32 crc32c( true ); -int cl_owner = -1; // global vars needed by add_member +int cl_owner = -1; // global vars needed by add_member int cl_group = -1; int cl_data_size = 0; -Solidity solidity = no_solid; +Solidity solidity = bsolid; namespace { -LZ_Encoder * encoder = 0; // local vars needed by add_member +LZ_Encoder * encoder = 0; // local vars needed by add_member const char * archive_namep = 0; -unsigned long long partial_data_size = 0; // current block size -int outfd = -1; -int gretval = 0; +unsigned long long partial_data_size = 0; // size of current block +Resizable_buffer grbuf( 2 * header_size ); // extended header + data +int goutfd = -1; +int error_status = 0; + +class File_is_the_archive + { + dev_t archive_dev; + ino_t archive_ino; + bool initialized; + +public: + File_is_the_archive() : initialized( false ) {} + bool init( const int fd ) + { + struct stat st; + if( fstat( fd, &st ) != 0 ) return false; + if( S_ISREG( st.st_mode ) ) + { archive_dev = st.st_dev; archive_ino = st.st_ino; initialized = true; } + return true; + } + bool operator()( const struct stat & st ) const + { + return initialized && archive_dev == st.st_dev && archive_ino == st.st_ino; + } + } file_is_the_archive; + + +bool option_C_after_relative_filename( const Arg_parser & parser ) + { + for( int i = 0; i < parser.arguments(); ++i ) + if( !parser.code( i ) && parser.argument( i ).size() && + parser.argument( i )[0] != '/' ) // relative_filename + while( ++i < parser.arguments() ) + if( parser.code( i ) == 'C' ) return true; + return false; + } + int seek_read( const int fd, uint8_t * const buf, const int size, const long long pos ) @@ -151,33 +186,14 @@ bool check_appendable( const int fd, const bool remove_eof ) } -class File_is_the_archive - { - dev_t archive_dev; - ino_t archive_ino; - bool initialized; - -public: - File_is_the_archive() : initialized( false ) {} - bool init( const int fd ) - { - struct stat st; - if( fstat( fd, &st ) != 0 ) return false; - if( S_ISREG( st.st_mode ) ) - { archive_dev = st.st_dev; archive_ino = st.st_ino; initialized = true; } - return true; - } - bool operator()( const struct stat & st ) const - { - return initialized && archive_dev == st.st_dev && archive_ino == st.st_ino; - } - } file_is_the_archive; - - bool archive_write( const uint8_t * const buf, const int size ) { + static bool flushed = true; // avoid flushing empty lzip members + + if( size <= 0 && flushed ) return true; + flushed = ( size <= 0 ); if( !encoder ) // uncompressed - return ( writeblock( outfd, buf, size ) == size ); + return ( writeblock( goutfd, buf, size ) == size ); enum { obuf_size = 65536 }; uint8_t obuf[obuf_size]; int sz = 0; @@ -191,7 +207,7 @@ bool archive_write( const uint8_t * const buf, const int size ) const int rd = LZ_compress_read( encoder, obuf, obuf_size ); if( rd < 0 ) internal_error( "library error (LZ_compress_read)." ); if( rd == 0 && sz >= size ) break; - if( writeblock( outfd, obuf, rd ) != rd ) return false; + if( writeblock( goutfd, obuf, rd ) != rd ) return false; } if( LZ_compress_finished( encoder ) == 1 && LZ_compress_restart_member( encoder, LLONG_MAX ) < 0 ) @@ -200,103 +216,17 @@ bool archive_write( const uint8_t * const buf, const int size ) } -void init_tar_header( Tar_header header ) // set magic and version - { - std::memset( header, 0, header_size ); - std::memcpy( header + magic_o, ustar_magic, magic_l - 1 ); - header[version_o] = header[version_o+1] = '0'; - } - - -unsigned char xdigit( const unsigned value ) - { - if( value <= 9 ) return '0' + value; - if( value <= 15 ) return 'A' + value - 10; - return 0; - } - -void print_hex( char * const buf, int size, unsigned long long num ) - { - while( --size >= 0 ) { buf[size] = xdigit( num & 0x0F ); num >>= 4; } - } - -void print_octal( uint8_t * const buf, int size, unsigned long long num ) - { - while( --size >= 0 ) { buf[size] = '0' + ( num % 8 ); num /= 8; } - } - bool write_extended( const Extended & extended ) { - const int path_rec = extended.recsize_path(); - const int lpath_rec = extended.recsize_linkpath(); - const int size_rec = extended.recsize_file_size(); - const unsigned long long edsize = extended.edsize(); - const unsigned long long bufsize = extended.edsize_pad(); - if( edsize >= 1ULL << 33 ) return false; // too much extended data - if( bufsize == 0 ) return edsize == 0; // overflow or no extended data - char * const buf = new char[bufsize+1]; // extended records buffer - unsigned long long pos = path_rec; // goto can't cross these - const unsigned crc_size = Extended::crc_record.size(); - - if( path_rec && snprintf( buf, path_rec + 1, "%d path=%s\n", - path_rec, extended.path().c_str() ) != path_rec ) - goto error; - if( lpath_rec && snprintf( buf + pos, lpath_rec + 1, "%d linkpath=%s\n", - lpath_rec, extended.linkpath().c_str() ) != lpath_rec ) - goto error; - pos += lpath_rec; - if( size_rec && snprintf( buf + pos, size_rec + 1, "%d size=%llu\n", - size_rec, extended.file_size() ) != size_rec ) - goto error; - pos += size_rec; - std::memcpy( buf + pos, Extended::crc_record.c_str(), crc_size ); - pos += crc_size; - if( pos != edsize ) goto error; - print_hex( buf + edsize - 9, 8, - crc32c.windowed_crc( (const uint8_t *)buf, edsize - 9, edsize ) ); - std::memset( buf + edsize, 0, bufsize - edsize ); // wipe padding - Tar_header header; // extended header - init_tar_header( header ); - header[typeflag_o] = tf_extended; // fill only required fields - print_octal( header + size_o, size_l - 1, edsize ); - print_octal( header + chksum_o, chksum_l - 1, ustar_chksum( header ) ); - if( !archive_write( header, header_size ) ) goto error; - for( pos = 0; pos < bufsize; ) // write extended records to archive + const long long ebsize = extended.format_block( grbuf ); + if( ebsize < 0 ) return false; + for( long long pos = 0; pos < ebsize; ) // write extended block to archive { - int size = std::min( bufsize - pos, 1ULL << 20 ); - if( !archive_write( (const uint8_t *)buf + pos, size ) ) goto error; + int size = std::min( ebsize - pos, 1LL << 20 ); + if( !archive_write( (const uint8_t *)grbuf() + pos, size ) ) return false; pos += size; } - delete[] buf; return true; -error: - delete[] buf; - return false; - } - - -const char * remove_leading_dotdot( const char * const filename ) - { - static std::string prefix; - const char * p = filename; - - for( int i = 0; filename[i]; ++i ) - if( filename[i] == '.' && filename[i+1] == '.' && - ( i == 0 || filename[i-1] == '/' ) && - ( filename[i+2] == 0 || filename[i+2] == '/' ) ) p = filename + i + 2; - while( *p == '/' || ( *p == '.' && p[1] == '/' ) ) ++p; - if( p != filename ) - { - std::string msg( filename, p - filename ); - if( prefix != msg ) - { - prefix = msg; - msg = "Removing leading '"; msg += prefix; msg += "' from member names."; - show_error( msg.c_str() ); - } - } - if( *p == 0 ) p = "."; - return p; } @@ -304,7 +234,7 @@ const char * remove_leading_dotdot( const char * const filename ) bool store_name( const char * const filename, Extended & extended, Tar_header header, const bool force_extended_name ) { - const char * const stored_name = remove_leading_dotdot( filename ); + const char * const stored_name = remove_leading_dotslash( filename, true ); if( !force_extended_name ) // try storing filename in the ustar header { @@ -327,18 +257,113 @@ bool store_name( const char * const filename, Extended & extended, } +// add one tar member to the archive int add_member( const char * const filename, const struct stat *, const int flag, struct FTW * ) { + unsigned long long file_size = 0; + Extended extended; // metadata for extended records + Tar_header header; + if( !fill_headers( filename, extended, header, file_size, flag ) ) return 0; + const int infd = file_size ? open_instream( filename ) : -1; + if( file_size && infd < 0 ) { set_error_status( 1 ); return 0; } + + if( encoder && solidity == bsolid && + block_is_full( extended, file_size, partial_data_size ) && + !archive_write( 0, 0 ) ) + { show_error( "Error flushing encoder", errno ); return 1; } + + if( !write_extended( extended ) ) + { show_error( "Error writing extended header", errno ); return 1; } + if( !archive_write( header, header_size ) ) + { show_error( "Error writing ustar header", errno ); return 1; } + if( file_size ) + { + enum { bufsize = 32 * header_size }; + uint8_t buf[bufsize]; + unsigned long long rest = file_size; + while( rest > 0 ) + { + int size = std::min( rest, (unsigned long long)bufsize ); + const int rd = readblock( infd, buf, size ); + rest -= rd; + if( rd != size ) + { + if( verbosity >= 0 ) + std::fprintf( stderr, "File '%s' ends unexpectedly at pos %llu\n", + filename, file_size - rest ); + close( infd ); return 1; + } + if( rest == 0 ) // last read + { + const int rem = file_size % header_size; + if( rem > 0 ) + { const int padding = header_size - rem; + std::memset( buf + size, 0, padding ); size += padding; } + } + if( !archive_write( buf, size ) ) + { show_error( "Error writing archive", errno ); close( infd ); + return 1; } + } + if( close( infd ) != 0 ) + { show_file_error( filename, "Error closing file", errno ); return 1; } + } + if( encoder && solidity == no_solid && !archive_write( 0, 0 ) ) + { show_error( "Error flushing encoder", errno ); return 1; } + if( verbosity >= 1 ) std::fprintf( stderr, "%s\n", filename ); + return 0; + } + +} // end namespace + + +/* Removes any amount of leading "./" and '/' strings from filename. + Optionally also removes prefixes containing a ".." component. */ +const char * remove_leading_dotslash( const char * const filename, + const bool dotdot ) + { + // prevent two threads from modifying the list of prefixes at the same time + static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + static std::vector< std::string > prefixes; // list of prefixes + const char * p = filename; + + if( dotdot ) + for( int i = 0; filename[i]; ++i ) + if( filename[i] == '.' && filename[i+1] == '.' && + ( i == 0 || filename[i-1] == '/' ) && + ( filename[i+2] == 0 || filename[i+2] == '/' ) ) + p = filename + i + 2; + while( *p == '/' || ( *p == '.' && p[1] == '/' ) ) ++p; + if( p != filename ) + { + std::string msg( filename, p - filename ); + unsigned i = 0; + xlock( &mutex ); + while( i < prefixes.size() && prefixes[i] != msg ) ++i; + if( i >= prefixes.size() ) + { + prefixes.push_back( msg ); + msg.insert( 0, "Removing leading '" ); msg += "' from member names."; + show_error( msg.c_str() ); + } + xunlock( &mutex ); + } + if( *p == 0 && *filename != 0 ) p = "."; + return p; + } + + +bool fill_headers( const char * const filename, Extended & extended, + Tar_header header, unsigned long long & file_size, + const int flag ) + { struct stat st; if( lstat( filename, &st ) != 0 ) { show_file_error( filename, "Can't stat input file", errno ); - gretval = 1; return 0; } + set_error_status( 1 ); return false; } if( file_is_the_archive( st ) ) { show_file_error( archive_namep, "File is the archive; not dumped." ); - return 0; } - Extended extended; // metadata for extended records - Tar_header header; + return false; } init_tar_header( header ); bool force_extended_name = false; @@ -350,15 +375,14 @@ int add_member( const char * const filename, const struct stat *, const gid_t gid = ( cl_group >= 0 ) ? (gid_t)cl_group : st.st_gid; if( uid >= 2 << 20 || gid >= 2 << 20 ) { show_file_error( filename, "uid or gid is larger than 2_097_151." ); - gretval = 1; return 0; } + set_error_status( 1 ); return false; } print_octal( header + uid_o, uid_l - 1, uid ); print_octal( header + gid_o, gid_l - 1, gid ); const long long mtime = st.st_mtime; // shut up gcc if( mtime < 0 || mtime >= 1LL << 33 ) { show_file_error( filename, "mtime is out of ustar range [0, 8_589_934_591]." ); - gretval = 1; return 0; } + set_error_status( 1 ); return false; } print_octal( header + mtime_o, mtime_l - 1, mtime ); - unsigned long long file_size = 0; Typeflag typeflag; if( S_ISREG( mode ) ) { typeflag = tf_regular; file_size = st.st_size; } else if( S_ISDIR( mode ) ) @@ -366,7 +390,7 @@ int add_member( const char * const filename, const struct stat *, typeflag = tf_directory; if( flag == FTW_DNR ) { show_file_error( filename, "Can't open directory", errno ); - gretval = 1; return 0; } + set_error_status( 1 ); return false; } } else if( S_ISLNK( mode ) ) { @@ -384,94 +408,74 @@ int add_member( const char * const filename, const struct stat *, } if( len != st.st_size ) { show_file_error( filename, "Error reading link", (len < 0) ? errno : 0 ); - gretval = 1; return 0; } + set_error_status( 1 ); return false; } } else if( S_ISCHR( mode ) || S_ISBLK( mode ) ) { typeflag = S_ISCHR( mode ) ? tf_chardev : tf_blockdev; if( major( st.st_dev ) >= 2 << 20 || minor( st.st_dev ) >= 2 << 20 ) { show_file_error( filename, "devmajor or devminor is larger than 2_097_151." ); - gretval = 1; return 0; } + set_error_status( 1 ); return false; } print_octal( header + devmajor_o, devmajor_l - 1, major( st.st_dev ) ); print_octal( header + devminor_o, devminor_l - 1, minor( st.st_dev ) ); } else if( S_ISFIFO( mode ) ) typeflag = tf_fifo; else { show_file_error( filename, "Unknown file type." ); - gretval = 2; return 0; } + set_error_status( 2 ); return false; } header[typeflag_o] = typeflag; + errno = 0; const struct passwd * const pw = getpwuid( uid ); if( pw && pw->pw_name ) std::strncpy( (char *)header + uname_o, pw->pw_name, uname_l - 1 ); + else { show_file_error( filename, "Can't read user name from database", errno ); + set_error_status( 1 ); } + errno = 0; const struct group * const gr = getgrgid( gid ); if( gr && gr->gr_name ) std::strncpy( (char *)header + gname_o, gr->gr_name, gname_l - 1 ); + else { show_file_error( filename, "Can't read group name from database", errno ); + set_error_status( 1 ); } if( file_size >= 1ULL << 33 ) { extended.file_size( file_size ); force_extended_name = true; } else print_octal( header + size_o, size_l - 1, file_size ); store_name( filename, extended, header, force_extended_name ); print_octal( header + chksum_o, chksum_l - 1, ustar_chksum( header ) ); + return true; + } - const int infd = file_size ? open_instream( filename ) : -1; - if( file_size && infd < 0 ) { gretval = 1; return 0; } - if( encoder && solidity == bsolid ) - { - const unsigned long long member_size = - header_size + extended.full_size() + round_up( file_size ); - const unsigned long long target_size = cl_data_size; - if( partial_data_size >= target_size || - ( partial_data_size >= min_data_size && - partial_data_size + member_size / 2 > target_size ) ) - { - partial_data_size = member_size; - if( !archive_write( 0, 0 ) ) - { show_error( "Error flushing encoder", errno ); return 1; } - } - else partial_data_size += member_size; - } - if( !extended.empty() && !write_extended( extended ) ) - { show_error( "Error writing extended header", errno ); return 1; } - if( !archive_write( header, header_size ) ) - { show_error( "Error writing ustar header", errno ); return 1; } - if( file_size ) - { - enum { bufsize = 32 * header_size }; - uint8_t buf[bufsize]; - unsigned long long rest = file_size; - while( rest > 0 ) - { - int size = std::min( rest, (unsigned long long)bufsize ); - const int rd = readblock( infd, buf, size ); - rest -= rd; - if( rd != size ) - { - if( verbosity >= 0 ) - std::fprintf( stderr, "File '%s' ends unexpectedly at pos %llu\n", - filename, file_size - rest ); - close( infd ); return 1; - } - if( rest == 0 ) // last read - { - const int rem = file_size % header_size; - if( rem > 0 ) - { const int padding = header_size - rem; - std::memset( buf + size, 0, padding ); size += padding; } - } - if( !archive_write( buf, size ) ) - { show_error( "Error writing archive", errno ); close( infd ); - return 1; } - } - if( close( infd ) != 0 ) - { show_file_error( filename, "Error closing file", errno ); return 1; } - } - if( encoder && solidity == no_solid && !archive_write( 0, 0 ) ) - { show_error( "Error flushing encoder", errno ); return 1; } - if( verbosity >= 1 ) std::fprintf( stderr, "%s\n", filename ); - return 0; +bool block_is_full( const Extended & extended, + const unsigned long long file_size, + unsigned long long & partial_data_size ) + { + const unsigned long long member_size = + header_size + extended.full_size() + round_up( file_size ); + const unsigned long long target_size = cl_data_size; + if( partial_data_size >= target_size || + ( partial_data_size >= min_data_size && + partial_data_size + member_size / 2 > target_size ) ) + { partial_data_size = member_size; return true; } + partial_data_size += member_size; return false; } -} // end namespace +void set_error_status( const int retval ) + { + // prevent two threads from modifying the error_status at the same time + static pthread_mutex_t mutex = PTHREAD_MUTEX_INITIALIZER; + + xlock( &mutex ); + if( error_status < retval ) error_status = retval; + xunlock( &mutex ); + } + +int final_exit_status( int retval ) + { + if( !retval && error_status ) + { show_error( "Exiting with failure status due to previous errors." ); + retval = error_status; } + return retval; + } unsigned ustar_chksum( const uint8_t * const header ) { @@ -495,7 +499,8 @@ int concatenate( const std::string & archive_name, const Arg_parser & parser, if( archive_name.empty() ) { show_error( "'--concatenate' is incompatible with '-f -'.", 0, true ); return 1; } - if( ( outfd = open_outstream( archive_name, false ) ) < 0 ) return 1; + const int outfd = open_outstream( archive_name, false ); + if( outfd < 0 ) return 1; if( !file_is_the_archive.init( outfd ) ) { show_file_error( archive_name.c_str(), "Can't stat", errno ); return 1; } @@ -503,6 +508,7 @@ int concatenate( const std::string & archive_name, const Arg_parser & parser, for( int i = 0; i < parser.arguments(); ++i ) // copy archives { if( parser.code( i ) ) continue; // skip options + if( parser.argument( i ).empty() ) continue; // skip empty names const char * const filename = parser.argument( i ).c_str(); const int infd = open_instream( filename ); if( infd < 0 ) @@ -531,7 +537,8 @@ int concatenate( const std::string & archive_name, const Arg_parser & parser, int encode( const std::string & archive_name, const Arg_parser & parser, - const int filenames, const int level, const bool append ) + const int filenames, const int level, const int num_workers, + const int debug_level, const bool append ) { struct Lzma_options { @@ -557,8 +564,8 @@ int encode( const std::string & archive_name, const Arg_parser & parser, if( !filenames ) { show_error( "Cowardly refusing to create an empty archive.", 0, true ); return 1; } - if( archive_name.empty() ) outfd = STDOUT_FILENO; - else if( ( outfd = open_outstream( archive_name ) ) < 0 ) return 1; + if( archive_name.empty() ) goutfd = STDOUT_FILENO; + else if( ( goutfd = open_outstream( archive_name ) ) < 0 ) return 1; } else { @@ -570,14 +577,14 @@ int encode( const std::string & archive_name, const Arg_parser & parser, if( !compressed ) { show_error( "'--append' is incompatible with '--uncompressed'.", 0, true ); return 1; } - if( ( outfd = open_outstream( archive_name, false ) ) < 0 ) return 1; - if( !check_appendable( outfd, true ) ) + if( ( goutfd = open_outstream( archive_name, false ) ) < 0 ) return 1; + if( !check_appendable( goutfd, true ) ) { show_error( "This does not look like an appendable tar.lz archive." ); return 2; } } archive_namep = archive_name.size() ? archive_name.c_str() : "(stdout)"; - if( !file_is_the_archive.init( outfd ) ) + if( !file_is_the_archive.init( goutfd ) ) { show_file_error( archive_namep, "Can't stat", errno ); return 1; } if( compressed ) @@ -588,12 +595,22 @@ int encode( const std::string & archive_name, const Arg_parser & parser, if( level == 0 ) cl_data_size = 1 << 20; else cl_data_size = 2 * dictionary_size; } + /* CWD is not per-thread; multi-threaded --create can't be used if a + -C option appears after a relative filename in the command line. */ + if( solidity != asolid && solidity != solid && num_workers > 0 && + !option_C_after_relative_filename( parser ) ) + { + // show_file_error( archive_namep, "Multi-threaded --create" ); + return encode_lz( archive_namep, parser, dictionary_size, + option_mapping[level].match_len_limit, num_workers, + goutfd, debug_level ); + } encoder = LZ_compress_open( dictionary_size, option_mapping[level].match_len_limit, LLONG_MAX ); if( !encoder || LZ_compress_errno( encoder ) != LZ_ok ) { if( !encoder || LZ_compress_errno( encoder ) == LZ_mem_error ) - show_error( "Not enough memory. Try a lower compression level." ); + show_error( mem_msg2 ); else internal_error( "invalid argument to encoder." ); return 1; @@ -601,7 +618,7 @@ int encode( const std::string & archive_name, const Arg_parser & parser, } int retval = 0; - for( int i = 0; i < parser.arguments(); ++i ) // write members + for( int i = 0; i < parser.arguments(); ++i ) // parse command line { const int code = parser.code( i ); const std::string & arg = parser.argument( i ); @@ -610,17 +627,18 @@ int encode( const std::string & archive_name, const Arg_parser & parser, { show_file_error( filename, "Error changing working directory", errno ); retval = 1; break; } if( code ) continue; // skip options + if( parser.argument( i ).empty() ) continue; // skip empty names std::string deslashed; // arg without trailing slashes unsigned len = arg.size(); while( len > 1 && arg[len-1] == '/' ) --len; if( len < arg.size() ) { deslashed.assign( arg, 0, len ); filename = deslashed.c_str(); } struct stat st; - if( lstat( filename, &st ) != 0 ) + if( lstat( filename, &st ) != 0 ) // filename from command line { show_file_error( filename, "Can't stat input file", errno ); - if( gretval < 1 ) gretval = 1; } + set_error_status( 1 ); } else if( ( retval = nftw( filename, add_member, 16, FTW_PHYS ) ) != 0 ) - break; // write error + break; // write error else if( encoder && solidity == dsolid && !archive_write( 0, 0 ) ) { show_error( "Error flushing encoder", errno ); retval = 1; } } @@ -630,7 +648,8 @@ int encode( const std::string & archive_name, const Arg_parser & parser, enum { bufsize = 2 * header_size }; uint8_t buf[bufsize]; std::memset( buf, 0, bufsize ); - if( encoder && ( solidity == asolid || solidity == bsolid ) && + if( encoder && + ( solidity == asolid || ( solidity == bsolid && partial_data_size ) ) && !archive_write( 0, 0 ) ) { show_error( "Error flushing encoder", errno ); retval = 1; } else if( !archive_write( buf, bufsize ) || @@ -640,12 +659,7 @@ int encode( const std::string & archive_name, const Arg_parser & parser, } if( encoder && LZ_compress_close( encoder ) < 0 ) { show_error( "LZ_compress_close failed." ); retval = 1; } - if( close( outfd ) != 0 && !retval ) + if( close( goutfd ) != 0 && !retval ) { show_error( "Error closing archive", errno ); retval = 1; } - if( retval && archive_name.size() && !append ) - std::remove( archive_name.c_str() ); - if( !retval && gretval ) - { show_error( "Exiting with failure status due to previous errors." ); - retval = gretval; } - return retval; + return final_exit_status( retval ); } |