Merging upstream version 0.14.

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 184 insertions, 61 deletions

diff --git a/create.cc b/create.cc
index cba638a..f70b99f 100644
--- a/create.cc
+++ b/create.cc
@@ -54,7 +54,7 @@ namespace {
 LZ_Encoder * encoder = 0;		// local vars needed by add_member
 const char * archive_namep = 0;
 unsigned long long partial_data_size = 0;	// size of current block
-Resizable_buffer grbuf( 2 * header_size );	// extended header + data
+Resizable_buffer grbuf;				// extended header + data
 int goutfd = -1;
 int error_status = 0;
 
@@ -123,56 +123,108 @@ bool copy_file( const int infd, const int outfd, const long long max_size = -1 )
   }
 
 
-/* Check archive type. If success, leave fd file pos at 0.
-   If remove_eof, leave fd file pos at beginning of the EOF blocks. */
-bool check_appendable( const int fd, const bool remove_eof )
+/* Check archive type. Return position of EOF blocks or -1 if failure.
+   If remove_eof, leave fd file pos at beginning of the EOF blocks.
+   Else, leave fd file pos at 0. */
+long long check_appendable( const int fd, const bool remove_eof )
   {
-  struct stat st;
-  if( fstat( fd, &st ) != 0 || !S_ISREG( st.st_mode ) ) return false;
-  if( lseek( fd, 0, SEEK_SET ) != 0 ) return false;
+  struct stat st;				// fd must be regular
+  if( fstat( fd, &st ) != 0 || !S_ISREG( st.st_mode ) ) return -1;
+  if( lseek( fd, 0, SEEK_SET ) != 0 ) return -1;
   enum { bufsize = header_size + ( header_size / 8 ) };
   uint8_t buf[bufsize];
-  int rd = readblock( fd, buf, bufsize );
-  if( rd == 0 && errno == 0 ) return true;	// append to empty archive
-  if( rd < min_member_size || ( rd != bufsize && errno ) ) return false;
+  const int rd = readblock( fd, buf, bufsize );
+  if( rd == 0 && errno == 0 ) return 0;		// append to empty archive
+  if( rd < min_member_size || ( rd != bufsize && errno ) ) return -1;
   const Lzip_header * const p = (const Lzip_header *)buf;	// shut up gcc
-  if( !p->verify_magic() || !p->verify_version() ) return false;
+  if( !p->verify_magic() || !p->verify_version() ) return -1;
   LZ_Decoder * decoder = LZ_decompress_open();	// decompress first header
   if( !decoder || LZ_decompress_errno( decoder ) != LZ_ok ||
       LZ_decompress_write( decoder, buf, rd ) != rd ||
-      ( rd = LZ_decompress_read( decoder, buf, header_size ) ) != header_size )
-    { LZ_decompress_close( decoder ); return false; }
+      LZ_decompress_read( decoder, buf, header_size ) != header_size )
+    { LZ_decompress_close( decoder ); return -1; }
   LZ_decompress_close( decoder );
-  const bool maybe_eof = ( buf[0] == 0 );
-  if( !verify_ustar_chksum( buf ) && !maybe_eof ) return false;
+  const bool maybe_eof = block_is_zero( buf, header_size );
+  if( !verify_ustar_chksum( buf ) && !maybe_eof ) return -1;
   const long long end = lseek( fd, 0, SEEK_END );
-  if( end < min_member_size ) return false;
+  if( end < min_member_size ) return -1;
 
   Lzip_trailer trailer;
   if( seek_read( fd, trailer.data, Lzip_trailer::size,
-                 end - Lzip_trailer::size ) != Lzip_trailer::size )
-    return false;
+                 end - Lzip_trailer::size ) != Lzip_trailer::size ) return -1;
   const long long member_size = trailer.member_size();
   if( member_size < min_member_size || member_size > end ||
-      ( maybe_eof && member_size != end ) ) return false;
+      ( maybe_eof && member_size != end ) ) return -1;
 
   Lzip_header header;
   if( seek_read( fd, header.data, Lzip_header::size,
-                 end - member_size ) != Lzip_header::size ) return false;
+                 end - member_size ) != Lzip_header::size ) return -1;
   if( !header.verify_magic() || !header.verify_version() ||
-      !isvalid_ds( header.dictionary_size() ) ) return false;
+      !isvalid_ds( header.dictionary_size() ) ) return -1;
 
   const unsigned long long data_size = trailer.data_size();
-  if( data_size < header_size || data_size > 32256 ) return false;
+  if( data_size < header_size || data_size > 32256 ) return -1;
   const unsigned data_crc = trailer.data_crc();
   const CRC32 crc32;
   uint32_t crc = 0xFFFFFFFFU;
   for( unsigned i = 0; i < data_size; ++i ) crc32.update_byte( crc, 0 );
   crc ^= 0xFFFFFFFFU;
-  if( crc != data_crc ) return false;
+  if( crc != data_crc ) return -1;
 
   const long long pos = remove_eof ? end - member_size : 0;
-  return ( lseek( fd, pos, SEEK_SET ) == pos );
+  if( lseek( fd, pos, SEEK_SET ) != pos ) return -1;
+  return end - member_size;
+  }
+
+
+/* Skip all tar headers. Return position of EOF blocks or -1 if failure.
+   If remove_eof, leave fd file pos at beginning of the EOF blocks.
+   Else, leave fd file pos at 0. */
+long long check_uncompressed_appendable( const int fd, const bool remove_eof )
+  {
+  struct stat st;			// fd must be regular
+  if( fstat( fd, &st ) != 0 || !S_ISREG( st.st_mode ) ) return -1;
+  if( lseek( fd, 0, SEEK_SET ) != 0 ) return -1;
+  if( st.st_size == 0 ) return 0;	// append to empty archive
+  long long eof_pos = 0;
+  Extended extended;			// metadata from extended records
+  Resizable_buffer rbuf;		// extended records buffer
+  bool prev_extended = false;		// prev header was extended
+  while( true )				// process one tar member per iteration
+    {
+    Tar_header header;
+    const int rd = readblock( fd, header, header_size );
+    if( rd == 0 && errno == 0 ) break;		// missing EOF blocks
+    if( rd != header_size ) return -1;
+    if( !verify_ustar_chksum( header ) )	// maybe EOF
+      { if( block_is_zero( header, header_size ) ) break; else return -1; }
+    const Typeflag typeflag = (Typeflag)header[typeflag_o];
+    if( typeflag == tf_extended || typeflag == tf_global )
+      {
+      if( prev_extended ) return -1;
+      const unsigned long long edsize = parse_octal( header + size_o, size_l );
+      const unsigned long long bufsize = round_up( edsize );
+      if( edsize == 0 || edsize >= 1ULL << 33 || bufsize >= INT_MAX )
+        return -1;			// overflow or no extended data
+      if( !rbuf.resize( bufsize ) ) return -1;
+      if( readblock( fd, (uint8_t *)rbuf(), bufsize ) != (int)bufsize )
+        return -1;
+      if( typeflag == tf_extended )
+        { if( !extended.parse( rbuf(), edsize, false ) ) return -1;
+          prev_extended = true; }
+      continue;
+      }
+    prev_extended = false;
+
+    eof_pos = lseek( fd, round_up( extended.get_file_size_and_reset( header ) ),
+                     SEEK_CUR );
+    if( eof_pos <= 0 ) return -1;
+    }
+
+  if( prev_extended ) return -1;
+  const long long pos = remove_eof ? eof_pos : 0;
+  if( lseek( fd, pos, SEEK_SET ) != pos ) return -1;
+  return eof_pos;
   }
 
 
@@ -251,6 +303,7 @@ bool store_name( const char * const filename, Extended & extended,
 int add_member( const char * const filename, const struct stat *,
                 const int flag, struct FTW * )
   {
+  if( Exclude::excluded( filename ) ) return 0;		// skip excluded
   unsigned long long file_size = 0;
   Extended extended;		// metadata for extended records
   Tar_header header;
@@ -310,6 +363,26 @@ bool writeblock_wrapper( const int outfd, const uint8_t * const buffer,
   }
 
 
+// write End-Of-Archive records
+bool write_eof_records( const int outfd, const bool compressed )
+  {
+  if( compressed )
+    {
+    enum { eof_member_size = 44 };
+    const uint8_t eof_member[eof_member_size] = {
+      0x4C, 0x5A, 0x49, 0x50, 0x01, 0x0C, 0x00, 0x00, 0x6F, 0xFD, 0xFF, 0xFF,
+      0xA3, 0xB7, 0x80, 0x0C, 0x82, 0xDB, 0xFF, 0xFF, 0x9F, 0xF0, 0x00, 0x00,
+      0x2E, 0xAF, 0xB5, 0xEF, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+      0x2C, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 };
+    return writeblock_wrapper( outfd, eof_member, eof_member_size );
+    }
+  enum { bufsize = 2 * header_size };
+  uint8_t buf[bufsize];
+  std::memset( buf, 0, bufsize );
+  return writeblock_wrapper( outfd, buf, bufsize );
+  }
+
+
 /* Removes any amount of leading "./" and '/' strings from filename.
    Optionally also removes prefixes containing a ".." component. */
 const char * remove_leading_dotslash( const char * const filename,
@@ -348,7 +421,7 @@ bool fill_headers( const char * const filename, Extended & extended,
                    const int flag )
   {
   struct stat st;
-  if( lstat( filename, &st ) != 0 )
+  if( hstat( filename, &st ) != 0 )
     { show_file_error( filename, "Can't stat input file", errno );
       set_error_status( 1 ); return false; }
   if( file_is_the_archive( st ) )
@@ -492,52 +565,95 @@ bool verify_ustar_chksum( const uint8_t * const header )
     ustar_chksum( header ) == parse_octal( header + chksum_o, chksum_l ) ); }
 
 
-int concatenate( const std::string & archive_name, const Arg_parser & parser,
+bool has_lz_ext( const std::string & name )
+  {
+  return ( name.size() > 3 &&
+           name.compare( name.size() - 3, 3, ".lz" ) == 0 ) ||
+         ( name.size() > 4 &&
+           name.compare( name.size() - 4, 4, ".tlz" ) == 0 );
+  }
+
+
+int concatenate( std::string archive_name, const Arg_parser & parser,
                  const int filenames )
   {
   if( !filenames )
     { if( verbosity >= 1 ) show_error( "Nothing to concatenate." ); return 0; }
-  if( archive_name.empty() )
-    { show_error( "'--concatenate' is incompatible with '-f -'.", 0, true );
-      return 1; }
-  const int outfd = open_outstream( archive_name, false );
+  const bool to_stdout = archive_name.empty();
+  const int outfd =
+    to_stdout ? STDOUT_FILENO : open_outstream( archive_name, false );
   if( outfd < 0 ) return 1;
-  if( !file_is_the_archive.init( outfd ) )
+  if( to_stdout ) archive_name = "(stdout)";
+  else if( !file_is_the_archive.init( outfd ) )
     { show_file_error( archive_name.c_str(), "Can't stat", errno ); return 1; }
+  int compressed;					// tri-state bool
+  if( to_stdout ) compressed = -1;			// unknown
+  else
+    {
+    compressed = has_lz_ext( archive_name );		// default value
+    long long pos = check_appendable( outfd, true );
+    if( pos > 0 ) compressed = true;
+    else if( pos < 0 )
+      {
+      pos = check_uncompressed_appendable( outfd, true );
+      if( pos > 0 ) compressed = false;
+      else if( pos < 0 )
+        { show_file_error( archive_name.c_str(), compressed ?
+            "This does not look like an appendable tar.lz archive." :
+            "This does not look like an appendable tar archive." );
+          return 2; }
+      }
+    }
 
   int retval = 0;
+  bool eof_pending = false;
   for( int i = 0; i < parser.arguments(); ++i )		// copy archives
     {
     if( parser.code( i ) ) continue;			// skip options
     if( parser.argument( i ).empty() ) continue;	// skip empty names
     const char * const filename = parser.argument( i ).c_str();
+    if( Exclude::excluded( filename ) ) continue;	// skip excluded
     const int infd = open_instream( filename );
     if( infd < 0 ) { retval = 1; break; }
-    if( !check_appendable( infd, false ) )
-      { show_file_error( filename, "Not an appendable tar.lz archive." );
-        close( infd ); retval = 2; break; }
     struct stat st;
-    if( fstat( infd, &st ) == 0 && file_is_the_archive( st ) )
+    if( !to_stdout && fstat( infd, &st ) == 0 && file_is_the_archive( st ) )
       { show_file_error( filename, "File is the archive; not concatenated." );
         close( infd ); continue; }
-    if( !check_appendable( outfd, true ) )
-      { show_error( "This does not look like an appendable tar.lz archive." );
+    long long size;
+    if( compressed < 0 )	// not initialized yet
+      {
+      if( ( size = check_appendable( infd, false ) ) > 0 ) compressed = true;
+      else if( ( size = check_uncompressed_appendable( infd, false ) ) > 0 )
+        compressed = false;
+      else { size = -1 ; compressed = has_lz_ext( filename ); }
+      }
+    else size = compressed ? check_appendable( infd, false ) :
+                             check_uncompressed_appendable( infd, false );
+    if( size < 0 )
+      { show_file_error( filename, compressed ?
+                                   "Not an appendable tar.lz archive." :
+                                   "Not an appendable tar archive." );
         close( infd ); retval = 2; break; }
-    if( !copy_file( infd, outfd ) || close( infd ) != 0 )
+    if( !copy_file( infd, outfd, size ) || close( infd ) != 0 )
       { show_file_error( filename, "Error copying archive", errno );
-        retval = 1; break; }
+        eof_pending = false; retval = 1; break; }
+    eof_pending = true;
     if( verbosity >= 1 ) std::fprintf( stderr, "%s\n", filename );
     }
 
+  if( eof_pending && !write_eof_records( outfd, compressed ) && !retval )
+    retval = 1;
   if( close( outfd ) != 0 && !retval )
-    { show_error( "Error closing archive", errno ); retval = 1; }
+    { show_file_error( archive_name.c_str(), "Error closing archive", errno );
+      retval = 1; }
   return retval;
   }
 
 
 int encode( const std::string & archive_name, const Arg_parser & parser,
             const int filenames, const int level, const int num_workers,
-            const int debug_level, const bool append )
+            const int out_slots, const int debug_level, const bool append,
+            const bool dereference )
   {
   struct Lzma_options
     {
@@ -558,28 +674,32 @@ int encode( const std::string & archive_name, const Arg_parser & parser,
     { 1 << 25, 273 } };		// -9
   const bool compressed = ( level >= 0 && level <= 9 );
 
-  if( !append )
+  if( archive_name.size() && !compressed && has_lz_ext( archive_name ) )
+    { show_file_error( archive_name.c_str(),
+      "Uncompressed mode incompatible with .lz extension." ); return 2; }
+
+  if( !filenames )
     {
-    if( !filenames )
+    if( !append && archive_name.size() )	// create archive
       { show_error( "Cowardly refusing to create an empty archive.", 0, true );
         return 1; }
-    if( archive_name.empty() ) goutfd = STDOUT_FILENO;
-    else if( ( goutfd = open_outstream( archive_name ) ) < 0 ) return 1;
+    else		// create/append to stdout or append to archive
+      { if( verbosity >= 1 ) show_error( "Nothing to append." ); return 0; }
     }
-  else
+
+  if( archive_name.empty() )			// create/append to stdout
+    goutfd = STDOUT_FILENO;
+  else if( !append )				// create archive
+    { if( ( goutfd = open_outstream( archive_name ) ) < 0 ) return 1; }
+  else						// append to archive
     {
-    if( !filenames )
-      { if( verbosity >= 1 ) show_error( "Nothing to append." ); return 0; }
-    if( archive_name.empty() )
-      { show_error( "'--append' is incompatible with '-f -'.", 0, true );
-        return 1; }
-    if( !compressed )
-      { show_error( "'--append' is incompatible with '--uncompressed'.", 0, true );
-        return 1; }
     if( ( goutfd = open_outstream( archive_name, false ) ) < 0 ) return 1;
-    if( !check_appendable( goutfd, true ) )
-      { show_error( "This does not look like an appendable tar.lz archive." );
-        return 2; }
+    if( compressed && check_appendable( goutfd, true ) < 0 )
+      { show_file_error( archive_name.c_str(),
+        "This does not look like an appendable tar.lz archive." ); return 2; }
+    if( !compressed && check_uncompressed_appendable( goutfd, true ) < 0 )
+      { show_file_error( archive_name.c_str(),
+        "This does not look like an appendable tar archive." ); return 2; }
     }
 
   archive_namep = archive_name.size() ? archive_name.c_str() : "(stdout)";
@@ -602,7 +722,7 @@ int encode( const std::string & archive_name, const Arg_parser & parser,
       // show_file_error( archive_namep, "Multi-threaded --create" );
       return encode_lz( parser, dictionary_size,
                         option_mapping[level].match_len_limit, num_workers,
-                        goutfd, debug_level );
+                        goutfd, out_slots, debug_level, dereference );
       }
     encoder = LZ_compress_open( dictionary_size,
                 option_mapping[level].match_len_limit, LLONG_MAX );
@@ -632,14 +752,16 @@ int encode( const std::string & archive_name, const Arg_parser & parser,
     while( len > 1 && arg[len-1] == '/' ) --len;
     if( len < arg.size() )
       { deslashed.assign( arg, 0, len ); filename = deslashed.c_str(); }
+    if( Exclude::excluded( filename ) ) continue;	// skip excluded
     struct stat st;
     if( lstat( filename, &st ) != 0 )	// filename from command line
       { show_file_error( filename, "Can't stat input file", errno );
         set_error_status( 1 ); }
-    else if( ( retval = nftw( filename, add_member, 16, FTW_PHYS ) ) != 0 )
+    else if( ( retval = nftw( filename, add_member, 16,
+                              dereference ? 0 : FTW_PHYS ) ) != 0 )
       break;					// write error
     else if( encoder && solidity == dsolid && !archive_write( 0, 0 ) )
-      retval = 1;
+      { retval = 1; break; }
     }
 
   if( !retval )			// write End-Of-Archive records
@@ -656,6 +778,7 @@ int encode( const std::string & archive_name, const Arg_parser & parser,
   if( encoder && LZ_compress_close( encoder ) < 0 )
     { show_error( "LZ_compress_close failed." ); retval = 1; }
   if( close( goutfd ) != 0 && !retval )
-    { show_error( "Error closing archive", errno ); retval = 1; }
+    { show_file_error( archive_name.c_str(), "Error closing archive", errno );
+      retval = 1; }
   return final_exit_status( retval );
   }