diff options
| author | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-15 09:41:34 +0000 |
|---|---|---|
| committer | Daniel Baumann <daniel.baumann@progress-linux.org> | 2024-06-15 09:41:34 +0000 |
| commit | 25e43e389c26d56f90f2f20e3cee19c808b2a18c (patch) | |
| tree | fdadca249ffd70901103b3f06a98aeab695c6193 /src | |
| parent | Adding debian version 5.6.1+really5.4.5-1. (diff) | |
| download | xz-utils-25e43e389c26d56f90f2f20e3cee19c808b2a18c.tar.xz xz-utils-25e43e389c26d56f90f2f20e3cee19c808b2a18c.zip | |
Merging upstream version 5.6.2.
Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>
Diffstat (limited to 'src')
241 files changed, 6298 insertions, 2479 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index d199e85..1061323 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## SUBDIRS = liblzma xzdec diff --git a/src/Makefile.in b/src/Makefile.in index d9827e9..73e7a2a 100644 --- a/src/Makefile.in +++ b/src/Makefile.in @@ -92,8 +92,8 @@ host_triplet = @host@ @COND_SCRIPTS_TRUE@am__append_3 = scripts subdir = src ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -206,7 +206,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -335,6 +334,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ diff --git a/src/common/common_w32res.rc b/src/common/common_w32res.rc index d05d22e..97aa58a 100644 --- a/src/common/common_w32res.rc +++ b/src/common/common_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #include <winresrc.h> diff --git a/src/common/mythread.h b/src/common/mythread.h index 4495e01..10ea2d4 100644 --- a/src/common/mythread.h +++ b/src/common/mythread.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file mythread.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef MYTHREAD_H @@ -112,6 +111,25 @@ mythread_sigmask(int how, const sigset_t *restrict set, # include <sys/time.h> #endif +// MinGW-w64 with winpthreads: +// +// NOTE: Typical builds with MinGW-w64 don't use this code (MYTHREAD_POSIX). +// Instead, native Windows threading APIs are used (MYTHREAD_VISTA or +// MYTHREAD_WIN95). +// +// MinGW-w64 has _sigset_t (an integer type) in <sys/types.h>. +// If _POSIX was #defined, the header would add the alias sigset_t too. +// Let's keep this working even without _POSIX. +// +// There are no functions that actually do something with sigset_t +// because signals barely exist on Windows. The sigfillset macro below +// is just to silence warnings. There is no sigfillset() in MinGW-w64. +#ifdef __MINGW32__ +# include <sys/types.h> +# define sigset_t _sigset_t +# define sigfillset(set_ptr) do { *(set_ptr) = 0; } while (0) +#endif + #define MYTHREAD_RET_TYPE void * #define MYTHREAD_RET_VALUE NULL @@ -140,11 +158,13 @@ typedef struct timespec mythread_condtime; // Use pthread_sigmask() to set the signal mask in multi-threaded programs. // Do nothing on OpenVMS since it lacks pthread_sigmask(). +// Do nothing on MinGW-w64 too to silence warnings (its pthread_sigmask() +// is #defined to 0 so it's a no-op). static inline void mythread_sigmask(int how, const sigset_t *restrict set, sigset_t *restrict oset) { -#ifdef __VMS +#if defined(__VMS) || defined(__MINGW32__) (void)how; (void)set; (void)oset; @@ -180,7 +200,7 @@ mythread_join(mythread thread) } -// Initiatlizes a mutex. Returns zero on success and non-zero on error. +// Initializes a mutex. Returns zero on success and non-zero on error. static inline int mythread_mutex_init(mythread_mutex *mutex) { diff --git a/src/common/sysdefs.h b/src/common/sysdefs.h index f04e45d..5f3785b 100644 --- a/src/common/sysdefs.h +++ b/src/common/sysdefs.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file sysdefs.h @@ -8,9 +10,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SYSDEFS_H @@ -159,13 +158,16 @@ typedef unsigned char _Bool; #include <string.h> -// As of MSVC 2013, inline and restrict are supported with -// non-standard keywords. -#if defined(_WIN32) && defined(_MSC_VER) -# ifndef inline +// Visual Studio 2013 update 2 supports only __inline, not inline. +// MSVC v19.0 / VS 2015 and newer support both. +// +// MSVC v19.27 (VS 2019 version 16.7) added support for restrict. +// Older ones support only __restrict. +#ifdef _MSC_VER +# if _MSC_VER < 1900 && !defined(inline) # define inline __inline # endif -# ifndef restrict +# if _MSC_VER < 1927 && !defined(restrict) # define restrict __restrict # endif #endif diff --git a/src/common/tuklib_common.h b/src/common/tuklib_common.h index b1f531e..7554dfc 100644 --- a/src/common/tuklib_common.h +++ b/src/common/tuklib_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_COMMON_H @@ -57,8 +56,28 @@ # define TUKLIB_GNUC_REQ(major, minor) 0 #endif -#if TUKLIB_GNUC_REQ(2, 5) +// tuklib_attr_noreturn attribute is used to mark functions as non-returning. +// We cannot use "noreturn" as the macro name because then C23 code that +// uses [[noreturn]] would break as it would expand to [[ [[noreturn]] ]]. +// +// tuklib_attr_noreturn must be used at the beginning of function declaration +// to work in all cases. The [[noreturn]] syntax is the most limiting, it +// must be even before any GNU C's __attribute__ keywords: +// +// tuklib_attr_noreturn +// __attribute__((nonnull(1))) +// extern void foo(const char *s); +// +// FIXME: Update __STDC_VERSION__ for the final C23 version. 202000 is used +// by GCC 13 and Clang 15 with -std=c2x. +#if defined(__STDC_VERSION__) && __STDC_VERSION__ >= 202000 +# define tuklib_attr_noreturn [[noreturn]] +#elif defined(__STDC_VERSION__) && __STDC_VERSION__ >= 201112 +# define tuklib_attr_noreturn _Noreturn +#elif TUKLIB_GNUC_REQ(2, 5) # define tuklib_attr_noreturn __attribute__((__noreturn__)) +#elif defined(_MSC_VER) +# define tuklib_attr_noreturn __declspec(noreturn) #else # define tuklib_attr_noreturn #endif diff --git a/src/common/tuklib_config.h b/src/common/tuklib_config.h index 9d470ba..b27251d 100644 --- a/src/common/tuklib_config.h +++ b/src/common/tuklib_config.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + // If config.h isn't available, assume that the headers required by // tuklib_common.h are available. This is required by crc32_tablegen.c. #ifdef HAVE_CONFIG_H diff --git a/src/common/tuklib_cpucores.c b/src/common/tuklib_cpucores.c index bb3f2f7..c4a781a 100644 --- a/src/common/tuklib_cpucores.c +++ b/src/common/tuklib_cpucores.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_cpucores.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "tuklib_cpucores.h" diff --git a/src/common/tuklib_cpucores.h b/src/common/tuklib_cpucores.h index be1ce1c..edff939 100644 --- a/src/common/tuklib_cpucores.h +++ b/src/common/tuklib_cpucores.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_cpucores.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_CPUCORES_H diff --git a/src/common/tuklib_exit.c b/src/common/tuklib_exit.c index aa55620..c84e0f6 100644 --- a/src/common/tuklib_exit.c +++ b/src/common/tuklib_exit.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_exit.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "tuklib_common.h" diff --git a/src/common/tuklib_exit.h b/src/common/tuklib_exit.h index b11776f..d4e6b4a 100644 --- a/src/common/tuklib_exit.h +++ b/src/common/tuklib_exit.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_exit.h @@ -6,9 +8,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_EXIT_H @@ -18,8 +17,8 @@ TUKLIB_DECLS_BEGIN #define tuklib_exit TUKLIB_SYMBOL(tuklib_exit) -extern void tuklib_exit(int status, int err_status, int show_error) - tuklib_attr_noreturn; +tuklib_attr_noreturn +extern void tuklib_exit(int status, int err_status, int show_error); TUKLIB_DECLS_END #endif diff --git a/src/common/tuklib_gettext.h b/src/common/tuklib_gettext.h index ff18904..3ef5cb7 100644 --- a/src/common/tuklib_gettext.h +++ b/src/common/tuklib_gettext.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_gettext.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_GETTEXT_H diff --git a/src/common/tuklib_integer.h b/src/common/tuklib_integer.h index e22aa8a..fbd5fb2 100644 --- a/src/common/tuklib_integer.h +++ b/src/common/tuklib_integer.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_integer.h @@ -14,7 +16,7 @@ /// /// Endianness-converting integer operations (these can be macros!) /// (XX = 16, 32, or 64; Y = b or l): -/// - Byte swapping: bswapXX(num) +/// - Byte swapping: byteswapXX(num) /// - Byte order conversions to/from native (byteswaps if Y isn't /// the native endianness): convXXYe(num) /// - Unaligned reads: readXXYe(ptr) @@ -37,9 +39,6 @@ // Authors: Lasse Collin // Joachim Henke // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_INTEGER_H @@ -67,38 +66,41 @@ #if defined(HAVE___BUILTIN_BSWAPXX) // GCC >= 4.8 and Clang -# define bswap16(n) __builtin_bswap16(n) -# define bswap32(n) __builtin_bswap32(n) -# define bswap64(n) __builtin_bswap64(n) +# define byteswap16(num) __builtin_bswap16(num) +# define byteswap32(num) __builtin_bswap32(num) +# define byteswap64(num) __builtin_bswap64(num) #elif defined(HAVE_BYTESWAP_H) // glibc, uClibc, dietlibc # include <byteswap.h> # ifdef HAVE_BSWAP_16 -# define bswap16(num) bswap_16(num) +# define byteswap16(num) bswap_16(num) # endif # ifdef HAVE_BSWAP_32 -# define bswap32(num) bswap_32(num) +# define byteswap32(num) bswap_32(num) # endif # ifdef HAVE_BSWAP_64 -# define bswap64(num) bswap_64(num) +# define byteswap64(num) bswap_64(num) # endif #elif defined(HAVE_SYS_ENDIAN_H) // *BSDs and Darwin # include <sys/endian.h> +# define byteswap16(num) bswap16(num) +# define byteswap32(num) bswap32(num) +# define byteswap64(num) bswap64(num) #elif defined(HAVE_SYS_BYTEORDER_H) // Solaris # include <sys/byteorder.h> # ifdef BSWAP_16 -# define bswap16(num) BSWAP_16(num) +# define byteswap16(num) BSWAP_16(num) # endif # ifdef BSWAP_32 -# define bswap32(num) BSWAP_32(num) +# define byteswap32(num) BSWAP_32(num) # endif # ifdef BSWAP_64 -# define bswap64(num) BSWAP_64(num) +# define byteswap64(num) BSWAP_64(num) # endif # ifdef BE_16 # define conv16be(num) BE_16(num) @@ -120,15 +122,15 @@ # endif #endif -#ifndef bswap16 -# define bswap16(n) (uint16_t)( \ +#ifndef byteswap16 +# define byteswap16(n) (uint16_t)( \ (((n) & 0x00FFU) << 8) \ | (((n) & 0xFF00U) >> 8) \ ) #endif -#ifndef bswap32 -# define bswap32(n) (uint32_t)( \ +#ifndef byteswap32 +# define byteswap32(n) (uint32_t)( \ (((n) & UINT32_C(0x000000FF)) << 24) \ | (((n) & UINT32_C(0x0000FF00)) << 8) \ | (((n) & UINT32_C(0x00FF0000)) >> 8) \ @@ -136,8 +138,8 @@ ) #endif -#ifndef bswap64 -# define bswap64(n) (uint64_t)( \ +#ifndef byteswap64 +# define byteswap64(n) (uint64_t)( \ (((n) & UINT64_C(0x00000000000000FF)) << 56) \ | (((n) & UINT64_C(0x000000000000FF00)) << 40) \ | (((n) & UINT64_C(0x0000000000FF0000)) << 24) \ @@ -161,23 +163,23 @@ # define conv64be(num) ((uint64_t)(num)) # endif # ifndef conv16le -# define conv16le(num) bswap16(num) +# define conv16le(num) byteswap16(num) # endif # ifndef conv32le -# define conv32le(num) bswap32(num) +# define conv32le(num) byteswap32(num) # endif # ifndef conv64le -# define conv64le(num) bswap64(num) +# define conv64le(num) byteswap64(num) # endif #else # ifndef conv16be -# define conv16be(num) bswap16(num) +# define conv16be(num) byteswap16(num) # endif # ifndef conv32be -# define conv32be(num) bswap32(num) +# define conv32be(num) byteswap32(num) # endif # ifndef conv64be -# define conv64be(num) bswap64(num) +# define conv64be(num) byteswap64(num) # endif # ifndef conv16le # define conv16le(num) ((uint16_t)(num)) @@ -251,7 +253,7 @@ // was one instruction longer. // // Conclusion: At least in case of GCC and Clang, byte-by-byte code is -// the best choise for strict-align archs to do unaligned access. +// the best choice for strict-align archs to do unaligned access. // // See also: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=111502 // @@ -625,7 +627,7 @@ write64le(uint8_t *buf, uint64_t num) // aligned but some compilers have language extensions to do that. With // such language extensions the memcpy() method gives excellent results. // -// What to do on a strict-align system when no known language extentensions +// What to do on a strict-align system when no known language extensions // are available? Falling back to byte-by-byte access would be safe but ruin // optimizations that have been made specifically with aligned access in mind. // As a compromise, aligned reads will fall back to non-compliant type punning diff --git a/src/common/tuklib_mbstr.h b/src/common/tuklib_mbstr.h index dde9305..4c8eeb7 100644 --- a/src/common/tuklib_mbstr.h +++ b/src/common/tuklib_mbstr.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_mbstr.h @@ -10,9 +12,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_MBSTR_H diff --git a/src/common/tuklib_mbstr_fw.c b/src/common/tuklib_mbstr_fw.c index 64c9ad5..22d883b 100644 --- a/src/common/tuklib_mbstr_fw.c +++ b/src/common/tuklib_mbstr_fw.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_mbstr_fw.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "tuklib_mbstr.h" diff --git a/src/common/tuklib_mbstr_width.c b/src/common/tuklib_mbstr_width.c index 69d159e..7a8bf07 100644 --- a/src/common/tuklib_mbstr_width.c +++ b/src/common/tuklib_mbstr_width.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_mbstr_width.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "tuklib_mbstr.h" diff --git a/src/common/tuklib_open_stdxxx.c b/src/common/tuklib_open_stdxxx.c index 26702a6..b93e61d 100644 --- a/src/common/tuklib_open_stdxxx.c +++ b/src/common/tuklib_open_stdxxx.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_open_stdxxx.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "tuklib_open_stdxxx.h" diff --git a/src/common/tuklib_open_stdxxx.h b/src/common/tuklib_open_stdxxx.h index b911616..3ee3ade 100644 --- a/src/common/tuklib_open_stdxxx.h +++ b/src/common/tuklib_open_stdxxx.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_open_stdxxx.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_OPEN_STDXXX_H diff --git a/src/common/tuklib_physmem.c b/src/common/tuklib_physmem.c index 69f6fd4..1009df1 100644 --- a/src/common/tuklib_physmem.c +++ b/src/common/tuklib_physmem.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_physmem.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "tuklib_physmem.h" @@ -73,23 +72,20 @@ #endif -// With GCC >= 8.1 with -Wextra and Clang >= 13 with -Wcast-function-type -// will warn about the Windows-specific code. -#if defined(__has_warning) -# if __has_warning("-Wcast-function-type") -# define CAN_DISABLE_WCAST_FUNCTION_TYPE 1 -# endif -#elif TUKLIB_GNUC_REQ(8,1) -# define CAN_DISABLE_WCAST_FUNCTION_TYPE 1 -#endif - - extern uint64_t tuklib_physmem(void) { uint64_t ret = 0; #if defined(_WIN32) || defined(__CYGWIN__) + // This requires Windows 2000 or later. + MEMORYSTATUSEX meminfo; + meminfo.dwLength = sizeof(meminfo); + if (GlobalMemoryStatusEx(&meminfo)) + ret = meminfo.ullTotalPhys; + +/* + // Old version that is compatible with even Win95: if ((GetVersion() & 0xFF) >= 5) { // Windows 2000 and later have GlobalMemoryStatusEx() which // supports reporting values greater than 4 GiB. To keep the @@ -125,6 +121,7 @@ tuklib_physmem(void) GlobalMemoryStatus(&meminfo); ret = meminfo.dwTotalPhys; } +*/ #elif defined(__OS2__) unsigned long mem; diff --git a/src/common/tuklib_physmem.h b/src/common/tuklib_physmem.h index 09e2a51..f35bfba 100644 --- a/src/common/tuklib_physmem.h +++ b/src/common/tuklib_physmem.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_physmem.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_PHYSMEM_H diff --git a/src/common/tuklib_progname.c b/src/common/tuklib_progname.c index e2ef4e5..959c127 100644 --- a/src/common/tuklib_progname.c +++ b/src/common/tuklib_progname.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_progname.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "tuklib_progname.h" diff --git a/src/common/tuklib_progname.h b/src/common/tuklib_progname.h index bb80f25..a3d90cb 100644 --- a/src/common/tuklib_progname.h +++ b/src/common/tuklib_progname.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file tuklib_progname.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef TUKLIB_PROGNAME_H diff --git a/src/liblzma/Makefile.am b/src/liblzma/Makefile.am index 23d524f..85b3a52 100644 --- a/src/liblzma/Makefile.am +++ b/src/liblzma/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## SUBDIRS = api @@ -24,7 +20,7 @@ liblzma_la_CPPFLAGS = \ -I$(top_srcdir)/src/liblzma/simple \ -I$(top_srcdir)/src/common \ -DTUKLIB_SYMBOL_PREFIX=lzma_ -liblzma_la_LDFLAGS = -no-undefined -version-info 9:5:4 +liblzma_la_LDFLAGS = -no-undefined -version-info 11:2:6 EXTRA_DIST += liblzma_generic.map liblzma_linux.map validate_map.sh if COND_SYMVERS_GENERIC diff --git a/src/liblzma/Makefile.in b/src/liblzma/Makefile.in index 2689fc4..45a8337 100644 --- a/src/liblzma/Makefile.in +++ b/src/liblzma/Makefile.in @@ -161,14 +161,14 @@ host_triplet = @host@ @COND_LZIP_DECODER_TRUE@@COND_MAIN_DECODER_TRUE@ common/lzip_decoder.c \ @COND_LZIP_DECODER_TRUE@@COND_MAIN_DECODER_TRUE@ common/lzip_decoder.h -@COND_CHECK_CRC32_TRUE@@COND_SMALL_TRUE@am__append_12 = check/crc32_small.c -@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@am__append_13 = \ -@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@ check/crc32_table.c \ -@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@ check/crc32_table_le.h \ -@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@ check/crc32_table_be.h - -@COND_ASM_X86_TRUE@@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@am__append_14 = check/crc32_x86.S -@COND_ASM_X86_FALSE@@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@am__append_15 = check/crc32_fast.c +@COND_SMALL_TRUE@am__append_12 = check/crc32_small.c +@COND_SMALL_FALSE@am__append_13 = \ +@COND_SMALL_FALSE@ check/crc32_table.c \ +@COND_SMALL_FALSE@ check/crc32_table_le.h \ +@COND_SMALL_FALSE@ check/crc32_table_be.h + +@COND_ASM_X86_TRUE@@COND_SMALL_FALSE@am__append_14 = check/crc32_x86.S +@COND_ASM_X86_FALSE@@COND_SMALL_FALSE@am__append_15 = check/crc32_fast.c @COND_CHECK_CRC64_TRUE@@COND_SMALL_TRUE@am__append_16 = check/crc64_small.c @COND_CHECK_CRC64_TRUE@@COND_SMALL_FALSE@am__append_17 = \ @COND_CHECK_CRC64_TRUE@@COND_SMALL_FALSE@ check/crc64_table.c \ @@ -256,14 +256,15 @@ host_triplet = @host@ @COND_FILTER_ARMTHUMB_TRUE@@COND_FILTER_SIMPLE_TRUE@am__append_43 = simple/armthumb.c @COND_FILTER_ARM64_TRUE@@COND_FILTER_SIMPLE_TRUE@am__append_44 = simple/arm64.c @COND_FILTER_SIMPLE_TRUE@@COND_FILTER_SPARC_TRUE@am__append_45 = simple/sparc.c -@COND_W32_TRUE@am__append_46 = liblzma.def liblzma.def.in empty.c -@COND_W32_TRUE@am__append_47 = liblzma_w32res.rc -@COND_W32_TRUE@am__append_48 = -Xlinker --output-def -Xlinker liblzma.def.in -@COND_SHARED_TRUE@@COND_W32_TRUE@am__append_49 = liblzma.def +@COND_FILTER_RISCV_TRUE@@COND_FILTER_SIMPLE_TRUE@am__append_46 = simple/riscv.c +@COND_W32_TRUE@am__append_47 = liblzma.def liblzma.def.in empty.c +@COND_W32_TRUE@am__append_48 = liblzma_w32res.rc +@COND_W32_TRUE@am__append_49 = -Xlinker --output-def -Xlinker liblzma.def.in +@COND_SHARED_TRUE@@COND_W32_TRUE@am__append_50 = liblzma.def subdir = src/liblzma ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -349,10 +350,10 @@ am__liblzma_la_SOURCES_DIST = ../common/tuklib_physmem.c \ common/vli_decoder.c common/stream_decoder_mt.c \ common/microlzma_decoder.c common/lzip_decoder.c \ common/lzip_decoder.h check/check.c check/check.h \ - check/crc_macros.h check/crc32_small.c check/crc32_table.c \ - check/crc32_table_le.h check/crc32_table_be.h \ - check/crc32_x86.S check/crc32_fast.c check/crc64_small.c \ - check/crc64_table.c check/crc64_table_le.h \ + check/crc_common.h check/crc_x86_clmul.h check/crc32_arm64.h \ + check/crc32_small.c check/crc32_table.c check/crc32_table_le.h \ + check/crc32_table_be.h check/crc32_x86.S check/crc32_fast.c \ + check/crc64_small.c check/crc64_table.c check/crc64_table_le.h \ check/crc64_table_be.h check/crc64_x86.S check/crc64_fast.c \ check/sha256.c lz/lz_encoder.c lz/lz_encoder.h \ lz/lz_encoder_hash.h lz/lz_encoder_hash_table.h \ @@ -374,7 +375,7 @@ am__liblzma_la_SOURCES_DIST = ../common/tuklib_physmem.c \ simple/simple_encoder.h simple/simple_decoder.c \ simple/simple_decoder.h simple/x86.c simple/powerpc.c \ simple/ia64.c simple/arm.c simple/armthumb.c simple/arm64.c \ - simple/sparc.c liblzma_w32res.rc + simple/sparc.c simple/riscv.c liblzma_w32res.rc @COND_THREADS_TRUE@am__objects_1 = liblzma_la-tuklib_cpucores.lo @COND_THREADS_TRUE@am__objects_2 = liblzma_la-hardware_cputhreads.lo \ @COND_THREADS_TRUE@ liblzma_la-outqueue.lo @@ -414,10 +415,11 @@ am__liblzma_la_SOURCES_DIST = ../common/tuklib_physmem.c \ @COND_MAIN_DECODER_TRUE@@COND_THREADS_TRUE@am__objects_7 = liblzma_la-stream_decoder_mt.lo @COND_MAIN_DECODER_TRUE@@COND_MICROLZMA_TRUE@am__objects_8 = liblzma_la-microlzma_decoder.lo @COND_LZIP_DECODER_TRUE@@COND_MAIN_DECODER_TRUE@am__objects_9 = liblzma_la-lzip_decoder.lo -@COND_CHECK_CRC32_TRUE@@COND_SMALL_TRUE@am__objects_10 = liblzma_la-crc32_small.lo -@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@am__objects_11 = liblzma_la-crc32_table.lo -@COND_ASM_X86_TRUE@@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@am__objects_12 = liblzma_la-crc32_x86.lo -@COND_ASM_X86_FALSE@@COND_CHECK_CRC32_TRUE@@COND_SMALL_FALSE@am__objects_13 = liblzma_la-crc32_fast.lo +@COND_SMALL_TRUE@am__objects_10 = liblzma_la-crc32_small.lo +@COND_SMALL_FALSE@am__objects_11 = liblzma_la-crc32_table.lo +@COND_ASM_X86_TRUE@@COND_SMALL_FALSE@am__objects_12 = \ +@COND_ASM_X86_TRUE@@COND_SMALL_FALSE@ liblzma_la-crc32_x86.lo +@COND_ASM_X86_FALSE@@COND_SMALL_FALSE@am__objects_13 = liblzma_la-crc32_fast.lo @COND_CHECK_CRC64_TRUE@@COND_SMALL_TRUE@am__objects_14 = liblzma_la-crc64_small.lo @COND_CHECK_CRC64_TRUE@@COND_SMALL_FALSE@am__objects_15 = liblzma_la-crc64_table.lo @COND_ASM_X86_TRUE@@COND_CHECK_CRC64_TRUE@@COND_SMALL_FALSE@am__objects_16 = liblzma_la-crc64_x86.lo @@ -450,7 +452,8 @@ am__objects_27 = @COND_FILTER_ARMTHUMB_TRUE@@COND_FILTER_SIMPLE_TRUE@am__objects_39 = liblzma_la-armthumb.lo @COND_FILTER_ARM64_TRUE@@COND_FILTER_SIMPLE_TRUE@am__objects_40 = liblzma_la-arm64.lo @COND_FILTER_SIMPLE_TRUE@@COND_FILTER_SPARC_TRUE@am__objects_41 = liblzma_la-sparc.lo -@COND_W32_TRUE@am__objects_42 = liblzma_w32res.lo +@COND_FILTER_RISCV_TRUE@@COND_FILTER_SIMPLE_TRUE@am__objects_42 = liblzma_la-riscv.lo +@COND_W32_TRUE@am__objects_43 = liblzma_w32res.lo am_liblzma_la_OBJECTS = liblzma_la-tuklib_physmem.lo $(am__objects_1) \ liblzma_la-common.lo liblzma_la-block_util.lo \ liblzma_la-easy_preset.lo liblzma_la-filter_common.lo \ @@ -471,7 +474,7 @@ am_liblzma_la_OBJECTS = liblzma_la-tuklib_physmem.lo $(am__objects_1) \ $(am__objects_33) $(am__objects_34) $(am__objects_35) \ $(am__objects_36) $(am__objects_37) $(am__objects_38) \ $(am__objects_39) $(am__objects_40) $(am__objects_41) \ - $(am__objects_42) + $(am__objects_42) $(am__objects_43) liblzma_la_OBJECTS = $(am_liblzma_la_OBJECTS) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) @@ -558,6 +561,7 @@ am__depfiles_remade = ./$(DEPDIR)/liblzma_la-alone_decoder.Plo \ ./$(DEPDIR)/liblzma_la-outqueue.Plo \ ./$(DEPDIR)/liblzma_la-powerpc.Plo \ ./$(DEPDIR)/liblzma_la-price_table.Plo \ + ./$(DEPDIR)/liblzma_la-riscv.Plo \ ./$(DEPDIR)/liblzma_la-sha256.Plo \ ./$(DEPDIR)/liblzma_la-simple_coder.Plo \ ./$(DEPDIR)/liblzma_la-simple_decoder.Plo \ @@ -691,7 +695,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -820,6 +823,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ @@ -842,8 +847,8 @@ SUBDIRS = api EXTRA_DIST = liblzma_generic.map liblzma_linux.map validate_map.sh \ check/crc32_tablegen.c check/crc64_tablegen.c $(am__append_23) \ liblzma.pc.in -CLEANFILES = $(am__append_46) -doc_DATA = $(am__append_49) +CLEANFILES = $(am__append_47) +doc_DATA = $(am__append_50) lib_LTLIBRARIES = liblzma.la liblzma_la_SOURCES = ../common/tuklib_physmem.c $(am__append_3) \ common/common.c common/common.h common/memcmplen.h \ @@ -855,18 +860,19 @@ liblzma_la_SOURCES = ../common/tuklib_physmem.c $(am__append_3) \ $(am__append_5) $(am__append_6) $(am__append_7) \ $(am__append_8) $(am__append_9) $(am__append_10) \ $(am__append_11) check/check.c check/check.h \ - check/crc_macros.h $(am__append_12) $(am__append_13) \ - $(am__append_14) $(am__append_15) $(am__append_16) \ - $(am__append_17) $(am__append_18) $(am__append_19) \ - $(am__append_20) $(am__append_21) $(am__append_22) \ - $(am__append_24) $(am__append_25) $(am__append_26) \ - $(am__append_27) $(am__append_28) $(am__append_29) \ - $(am__append_30) $(am__append_31) $(am__append_32) \ - $(am__append_33) $(am__append_34) $(am__append_35) \ - $(am__append_36) $(am__append_37) $(am__append_38) \ - $(am__append_39) $(am__append_40) $(am__append_41) \ - $(am__append_42) $(am__append_43) $(am__append_44) \ - $(am__append_45) $(am__append_47) + check/crc_common.h check/crc_x86_clmul.h check/crc32_arm64.h \ + $(am__append_12) $(am__append_13) $(am__append_14) \ + $(am__append_15) $(am__append_16) $(am__append_17) \ + $(am__append_18) $(am__append_19) $(am__append_20) \ + $(am__append_21) $(am__append_22) $(am__append_24) \ + $(am__append_25) $(am__append_26) $(am__append_27) \ + $(am__append_28) $(am__append_29) $(am__append_30) \ + $(am__append_31) $(am__append_32) $(am__append_33) \ + $(am__append_34) $(am__append_35) $(am__append_36) \ + $(am__append_37) $(am__append_38) $(am__append_39) \ + $(am__append_40) $(am__append_41) $(am__append_42) \ + $(am__append_43) $(am__append_44) $(am__append_45) \ + $(am__append_46) $(am__append_48) liblzma_la_CPPFLAGS = \ -I$(top_srcdir)/src/liblzma/api \ -I$(top_srcdir)/src/liblzma/common \ @@ -879,8 +885,8 @@ liblzma_la_CPPFLAGS = \ -I$(top_srcdir)/src/common \ -DTUKLIB_SYMBOL_PREFIX=lzma_ -liblzma_la_LDFLAGS = -no-undefined -version-info 9:5:4 $(am__append_1) \ - $(am__append_2) $(am__append_48) +liblzma_la_LDFLAGS = -no-undefined -version-info 11:2:6 \ + $(am__append_1) $(am__append_2) $(am__append_49) pkgconfigdir = $(libdir)/pkgconfig pkgconfig_DATA = liblzma.pc pc_verbose = $(pc_verbose_@AM_V@) @@ -1028,6 +1034,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/liblzma_la-outqueue.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/liblzma_la-powerpc.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/liblzma_la-price_table.Plo@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/liblzma_la-riscv.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/liblzma_la-sha256.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/liblzma_la-simple_coder.Plo@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/liblzma_la-simple_decoder.Plo@am__quote@ # am--include-marker @@ -1686,6 +1693,13 @@ liblzma_la-sparc.lo: simple/sparc.c @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(liblzma_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o liblzma_la-sparc.lo `test -f 'simple/sparc.c' || echo '$(srcdir)/'`simple/sparc.c +liblzma_la-riscv.lo: simple/riscv.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(liblzma_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT liblzma_la-riscv.lo -MD -MP -MF $(DEPDIR)/liblzma_la-riscv.Tpo -c -o liblzma_la-riscv.lo `test -f 'simple/riscv.c' || echo '$(srcdir)/'`simple/riscv.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/liblzma_la-riscv.Tpo $(DEPDIR)/liblzma_la-riscv.Plo +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='simple/riscv.c' object='liblzma_la-riscv.lo' libtool=yes @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(LIBTOOL) $(AM_V_lt) --tag=CC $(AM_LIBTOOLFLAGS) $(LIBTOOLFLAGS) --mode=compile $(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(liblzma_la_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o liblzma_la-riscv.lo `test -f 'simple/riscv.c' || echo '$(srcdir)/'`simple/riscv.c + mostlyclean-libtool: -rm -f *.lo @@ -1998,6 +2012,7 @@ distclean: distclean-recursive -rm -f ./$(DEPDIR)/liblzma_la-outqueue.Plo -rm -f ./$(DEPDIR)/liblzma_la-powerpc.Plo -rm -f ./$(DEPDIR)/liblzma_la-price_table.Plo + -rm -f ./$(DEPDIR)/liblzma_la-riscv.Plo -rm -f ./$(DEPDIR)/liblzma_la-sha256.Plo -rm -f ./$(DEPDIR)/liblzma_la-simple_coder.Plo -rm -f ./$(DEPDIR)/liblzma_la-simple_decoder.Plo @@ -2127,6 +2142,7 @@ maintainer-clean: maintainer-clean-recursive -rm -f ./$(DEPDIR)/liblzma_la-outqueue.Plo -rm -f ./$(DEPDIR)/liblzma_la-powerpc.Plo -rm -f ./$(DEPDIR)/liblzma_la-price_table.Plo + -rm -f ./$(DEPDIR)/liblzma_la-riscv.Plo -rm -f ./$(DEPDIR)/liblzma_la-sha256.Plo -rm -f ./$(DEPDIR)/liblzma_la-simple_coder.Plo -rm -f ./$(DEPDIR)/liblzma_la-simple_decoder.Plo diff --git a/src/liblzma/api/Makefile.am b/src/liblzma/api/Makefile.am index db24600..4f91c77 100644 --- a/src/liblzma/api/Makefile.am +++ b/src/liblzma/api/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## nobase_include_HEADERS = \ lzma.h \ @@ -21,3 +17,22 @@ nobase_include_HEADERS = \ lzma/stream_flags.h \ lzma/version.h \ lzma/vli.h + +if COND_DOXYGEN +$(top_builddir)/doc/api/index.html: $(top_srcdir)/doxygen/update-doxygen $(top_srcdir)/doxygen/Doxyfile $(nobase_include_HEADERS) + $(MKDIR_P) "$(top_builddir)/doc" + "$(top_srcdir)/doxygen/update-doxygen" api \ + "$(top_srcdir)" "$(top_builddir)/doc" + +all-local: $(top_builddir)/doc/api/index.html + +install-data-local: + $(MKDIR_P) "$(DESTDIR)$(docdir)/api" + $(INSTALL_DATA) "$(top_builddir)"/doc/api/* "$(DESTDIR)$(docdir)/api" + +uninstall-local: + rm -rf "$(DESTDIR)$(docdir)/api" + +clean-local: + rm -rf "$(top_builddir)/doc/api" +endif diff --git a/src/liblzma/api/Makefile.in b/src/liblzma/api/Makefile.in index ea5cfd6..e973884 100644 --- a/src/liblzma/api/Makefile.in +++ b/src/liblzma/api/Makefile.in @@ -90,8 +90,8 @@ build_triplet = @build@ host_triplet = @host@ subdir = src/liblzma/api ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -192,7 +192,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -321,6 +320,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ @@ -505,7 +506,8 @@ distdir-am: $(DISTFILES) done check-am: all-am check: check-am -all-am: Makefile $(HEADERS) +@COND_DOXYGEN_FALSE@all-local: +all-am: Makefile $(HEADERS) all-local installdirs: for dir in "$(DESTDIR)$(includedir)"; do \ test -z "$$dir" || $(MKDIR_P) "$$dir"; \ @@ -540,9 +542,12 @@ distclean-generic: maintainer-clean-generic: @echo "This command is intended for maintainers to use" @echo "it deletes files that may require special tools to rebuild." +@COND_DOXYGEN_FALSE@clean-local: +@COND_DOXYGEN_FALSE@install-data-local: +@COND_DOXYGEN_FALSE@uninstall-local: clean: clean-am -clean-am: clean-generic clean-libtool mostlyclean-am +clean-am: clean-generic clean-libtool clean-local mostlyclean-am distclean: distclean-am -rm -f Makefile @@ -560,7 +565,7 @@ info: info-am info-am: -install-data-am: install-nobase_includeHEADERS +install-data-am: install-data-local install-nobase_includeHEADERS install-dvi: install-dvi-am @@ -604,27 +609,45 @@ ps: ps-am ps-am: -uninstall-am: uninstall-nobase_includeHEADERS +uninstall-am: uninstall-local uninstall-nobase_includeHEADERS .MAKE: install-am install-strip -.PHONY: CTAGS GTAGS TAGS all all-am check check-am clean clean-generic \ - clean-libtool cscopelist-am ctags ctags-am distclean \ - distclean-generic distclean-libtool distclean-tags distdir dvi \ - dvi-am html html-am info info-am install install-am \ - install-data install-data-am install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man \ - install-nobase_includeHEADERS install-pdf install-pdf-am \ - install-ps install-ps-am install-strip installcheck \ - installcheck-am installdirs maintainer-clean \ - maintainer-clean-generic mostlyclean mostlyclean-generic \ - mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ - uninstall-am uninstall-nobase_includeHEADERS +.PHONY: CTAGS GTAGS TAGS all all-am all-local check check-am clean \ + clean-generic clean-libtool clean-local cscopelist-am ctags \ + ctags-am distclean distclean-generic distclean-libtool \ + distclean-tags distdir dvi dvi-am html html-am info info-am \ + install install-am install-data install-data-am \ + install-data-local install-dvi install-dvi-am install-exec \ + install-exec-am install-html install-html-am install-info \ + install-info-am install-man install-nobase_includeHEADERS \ + install-pdf install-pdf-am install-ps install-ps-am \ + install-strip installcheck installcheck-am installdirs \ + maintainer-clean maintainer-clean-generic mostlyclean \ + mostlyclean-generic mostlyclean-libtool pdf pdf-am ps ps-am \ + tags tags-am uninstall uninstall-am uninstall-local \ + uninstall-nobase_includeHEADERS .PRECIOUS: Makefile +@COND_DOXYGEN_TRUE@$(top_builddir)/doc/api/index.html: $(top_srcdir)/doxygen/update-doxygen $(top_srcdir)/doxygen/Doxyfile $(nobase_include_HEADERS) +@COND_DOXYGEN_TRUE@ $(MKDIR_P) "$(top_builddir)/doc" +@COND_DOXYGEN_TRUE@ "$(top_srcdir)/doxygen/update-doxygen" api \ +@COND_DOXYGEN_TRUE@ "$(top_srcdir)" "$(top_builddir)/doc" + +@COND_DOXYGEN_TRUE@all-local: $(top_builddir)/doc/api/index.html + +@COND_DOXYGEN_TRUE@install-data-local: +@COND_DOXYGEN_TRUE@ $(MKDIR_P) "$(DESTDIR)$(docdir)/api" +@COND_DOXYGEN_TRUE@ $(INSTALL_DATA) "$(top_builddir)"/doc/api/* "$(DESTDIR)$(docdir)/api" + +@COND_DOXYGEN_TRUE@uninstall-local: +@COND_DOXYGEN_TRUE@ rm -rf "$(DESTDIR)$(docdir)/api" + +@COND_DOXYGEN_TRUE@clean-local: +@COND_DOXYGEN_TRUE@ rm -rf "$(top_builddir)/doc/api" + # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: diff --git a/src/liblzma/api/lzma.h b/src/liblzma/api/lzma.h index de12f22..6ca6e50 100644 --- a/src/liblzma/api/lzma.h +++ b/src/liblzma/api/lzma.h @@ -1,31 +1,30 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file api/lzma.h * \brief The public API of liblzma data compression library * \mainpage * - * liblzma is a public domain general-purpose data compression library with - * a zlib-like API. The native file format is .xz, but also the old .lzma - * format and raw (no headers) streams are supported. Multiple compression - * algorithms (filters) are supported. Currently LZMA2 is the primary filter. + * liblzma is a general-purpose data compression library with a zlib-like API. + * The native file format is .xz, but also the old .lzma format and raw (no + * headers) streams are supported. Multiple compression algorithms (filters) + * are supported. Currently LZMA2 is the primary filter. + * + * liblzma is part of XZ Utils <https://tukaani.org/xz/>. XZ Utils + * includes a gzip-like command line tool named xz and some other tools. + * XZ Utils is developed and maintained by Lasse Collin. * - * liblzma is part of XZ Utils <https://tukaani.org/xz/>. XZ Utils includes - * a gzip-like command line tool named xz and some other tools. XZ Utils - * is developed and maintained by Lasse Collin and Jia Tan. + * Major parts of liblzma are based on code written by Igor Pavlov, + * specifically the LZMA SDK <https://7-zip.org/sdk.html>. * - * Major parts of liblzma are based on Igor Pavlov's public domain LZMA SDK - * <https://7-zip.org/sdk.html>. + * The SHA-256 implementation in liblzma is based on code written by + * Wei Dai in Crypto++ Library <https://www.cryptopp.com/>. * - * The SHA-256 implementation is based on the public domain code found from - * 7-Zip <https://7-zip.org/>, which has a modified version of the public - * domain SHA-256 code found from Crypto++ <https://www.cryptopp.com/>. - * The SHA-256 code in Crypto++ was written by Kevin Springle and Wei Dai. + * liblzma is distributed under the BSD Zero Clause License (0BSD). */ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H diff --git a/src/liblzma/api/lzma/base.h b/src/liblzma/api/lzma/base.h index 75cdd72..590e1d2 100644 --- a/src/liblzma/api/lzma/base.h +++ b/src/liblzma/api/lzma/base.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/base.h * \brief Data types and functions used in many places in liblzma API @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -21,8 +20,8 @@ * * This is here because C89 doesn't have stdbool.h. To set a value for * variables having type lzma_bool, you can use - * - C99's `true' and `false' from stdbool.h; - * - C++'s internal `true' and `false'; or + * - C99's 'true' and 'false' from stdbool.h; + * - C++'s internal 'true' and 'false'; or * - integers one (true) and zero (false). */ typedef unsigned char lzma_bool; @@ -258,7 +257,7 @@ typedef enum { */ /* - * These eumerations may be used internally by liblzma + * These enumerations may be used internally by liblzma * but they will never be returned to applications. */ LZMA_RET_INTERNAL1 = 101, @@ -273,13 +272,13 @@ typedef enum { /** - * \brief The `action' argument for lzma_code() + * \brief The 'action' argument for lzma_code() * * After the first use of LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, LZMA_FULL_BARRIER, - * or LZMA_FINISH, the same `action' must be used until lzma_code() returns + * or LZMA_FINISH, the same 'action' must be used until lzma_code() returns * LZMA_STREAM_END. Also, the amount of input (that is, strm->avail_in) must * not be modified by the application until lzma_code() returns - * LZMA_STREAM_END. Changing the `action' or modifying the amount of input + * LZMA_STREAM_END. Changing the 'action' or modifying the amount of input * will make lzma_code() return LZMA_PROG_ERROR. */ typedef enum { @@ -393,8 +392,8 @@ typedef enum { * Single-threaded mode only: liblzma doesn't make an internal copy of * lzma_allocator. Thus, it is OK to change these function pointers in * the middle of the coding process, but obviously it must be done - * carefully to make sure that the replacement `free' can deallocate - * memory allocated by the earlier `alloc' function(s). + * carefully to make sure that the replacement 'free' can deallocate + * memory allocated by the earlier 'alloc' function(s). * * Multithreaded mode: liblzma might internally store pointers to the * lzma_allocator given via the lzma_stream structure. The application @@ -422,7 +421,7 @@ typedef struct { * liblzma never sets this to zero. * * \return Pointer to the beginning of a memory block of - * `size' bytes, or NULL if allocation fails + * 'size' bytes, or NULL if allocation fails * for some reason. When allocation fails, functions * of liblzma return LZMA_MEM_ERROR. * @@ -622,7 +621,7 @@ typedef struct { * to and get output from liblzma. * * See the description of the coder-specific initialization function to find - * out what `action' values are supported by the coder. + * out what 'action' values are supported by the coder. * * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. diff --git a/src/liblzma/api/lzma/bcj.h b/src/liblzma/api/lzma/bcj.h index 0c84e0c..7f6611f 100644 --- a/src/liblzma/api/lzma/bcj.h +++ b/src/liblzma/api/lzma/bcj.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/bcj.h * \brief Branch/Call/Jump conversion filters @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -53,6 +52,11 @@ */ #define LZMA_FILTER_ARM64 LZMA_VLI_C(0x0A) +/** + * \brief Filter for RISC-V binaries + */ +#define LZMA_FILTER_RISCV LZMA_VLI_C(0x0B) + /** * \brief Options for BCJ filters diff --git a/src/liblzma/api/lzma/block.h b/src/liblzma/api/lzma/block.h index ec5e77a..05b77e5 100644 --- a/src/liblzma/api/lzma/block.h +++ b/src/liblzma/api/lzma/block.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/block.h * \brief .xz Block handling @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/check.h b/src/liblzma/api/lzma/check.h index b37197d..e7a50ed 100644 --- a/src/liblzma/api/lzma/check.h +++ b/src/liblzma/api/lzma/check.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/check.h * \brief Integrity checks @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/container.h b/src/liblzma/api/lzma/container.h index 2849fbf..8e4af42 100644 --- a/src/liblzma/api/lzma/container.h +++ b/src/liblzma/api/lzma/container.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/container.h * \brief File formats @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -297,7 +296,7 @@ extern LZMA_API(uint64_t) lzma_easy_decoder_memusage(uint32_t preset) * to call lzma_end() after failed initialization. * * If initialization succeeds, use lzma_code() to do the actual encoding. - * Valid values for `action' (the second argument of lzma_code()) are + * Valid values for 'action' (the second argument of lzma_code()) are * LZMA_RUN, LZMA_SYNC_FLUSH, LZMA_FULL_FLUSH, and LZMA_FINISH. In future, * there may be compression levels or flags that don't support LZMA_SYNC_FLUSH. * @@ -436,6 +435,34 @@ extern LZMA_API(lzma_ret) lzma_stream_encoder_mt( /** + * \brief Calculate recommended Block size for multithreaded .xz encoder + * + * This calculates a recommended Block size for multithreaded encoding given + * a filter chain. This is used internally by lzma_stream_encoder_mt() to + * determine the Block size if the block_size member is not set to the + * special value of 0 in the lzma_mt options struct. + * + * If one wishes to change the filters between Blocks, this function is + * helpful to set the block_size member of the lzma_mt struct before calling + * lzma_stream_encoder_mt(). Since the block_size member represents the + * maximum possible Block size for the multithreaded .xz encoder, one can + * use this function to find the maximum recommended Block size based on + * all planned filter chains. Otherwise, the multithreaded encoder will + * base its maximum Block size on the first filter chain used (if the + * block_size member is not set), which may unnecessarily limit the Block + * size for a later filter chain. + * + * \param filters Array of filters terminated with + * .id == LZMA_VLI_UNKNOWN. + * + * \return Recommended Block size in bytes, or UINT64_MAX if + * an error occurred. + */ +extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) + lzma_nothrow; + + +/** * \brief Initialize .lzma encoder (legacy file format) * * The .lzma format is sometimes called the LZMA_Alone format, which is the @@ -651,13 +678,13 @@ extern LZMA_API(lzma_ret) lzma_microlzma_encoder( * supported by liblzma, only the .xz and .lz formats allow concatenated * files. Concatenated files are not allowed with the legacy .lzma format. * - * This flag also affects the usage of the `action' argument for lzma_code(). + * This flag also affects the usage of the 'action' argument for lzma_code(). * When LZMA_CONCATENATED is used, lzma_code() won't return LZMA_STREAM_END - * unless LZMA_FINISH is used as `action'. Thus, the application has to set + * unless LZMA_FINISH is used as 'action'. Thus, the application has to set * LZMA_FINISH in the same way as it does when encoding. * * If LZMA_CONCATENATED is not used, the decoders still accept LZMA_FINISH - * as `action' for lzma_code(), but the usage of LZMA_FINISH isn't required. + * as 'action' for lzma_code(), but the usage of LZMA_FINISH isn't required. */ #define LZMA_CONCATENATED UINT32_C(0x08) @@ -765,7 +792,7 @@ extern LZMA_API(lzma_ret) lzma_stream_decoder_mt( * as it doesn't support any decoder flags. It will return LZMA_STREAM_END * after one .lzma stream.) * - * \param strm Pointer to lzma_stream that is at least initialized + * \param strm Pointer to lzma_stream that is at least initialized * with LZMA_STREAM_INIT. * \param memlimit Memory usage limit as bytes. Use UINT64_MAX * to effectively disable the limiter. liblzma @@ -791,7 +818,7 @@ extern LZMA_API(lzma_ret) lzma_auto_decoder( /** * \brief Initialize .lzma decoder (legacy file format) * - * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. * There is no need to use LZMA_FINISH, but it's allowed because it may * simplify certain types of applications. * diff --git a/src/liblzma/api/lzma/delta.h b/src/liblzma/api/lzma/delta.h index 7a725bc..5ebacef 100644 --- a/src/liblzma/api/lzma/delta.h +++ b/src/liblzma/api/lzma/delta.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/delta.h * \brief Delta filter @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/filter.h b/src/liblzma/api/lzma/filter.h index 1d887b4..e86809c 100644 --- a/src/liblzma/api/lzma/filter.h +++ b/src/liblzma/api/lzma/filter.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/filter.h * \brief Common filter related types and functions @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -43,7 +42,7 @@ typedef struct { /** * \brief Filter ID * - * Use constants whose name begin with `LZMA_FILTER_' to specify + * Use constants whose name begin with 'LZMA_FILTER_' to specify * different filters. In an array of lzma_filter structures, use * LZMA_VLI_UNKNOWN to indicate end of filters. * @@ -199,7 +198,7 @@ extern LZMA_API(uint64_t) lzma_raw_decoder_memusage(const lzma_filter *filters) * * This function may be useful when implementing custom file formats. * - * The `action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the + * The 'action' with lzma_code() can be LZMA_RUN, LZMA_SYNC_FLUSH (if the * filter chain supports it), or LZMA_FINISH. * * \param strm Pointer to lzma_stream that is at least @@ -223,7 +222,7 @@ extern LZMA_API(lzma_ret) lzma_raw_encoder( * * The initialization of raw decoder goes similarly to raw encoder. * - * The `action' with lzma_code() can be LZMA_RUN or LZMA_FINISH. Using + * The 'action' with lzma_code() can be LZMA_RUN or LZMA_FINISH. Using * LZMA_FINISH is not required, it is supported just for convenience. * * \param strm Pointer to lzma_stream that is at least diff --git a/src/liblzma/api/lzma/hardware.h b/src/liblzma/api/lzma/hardware.h index f34897d..7a1a84f 100644 --- a/src/liblzma/api/lzma/hardware.h +++ b/src/liblzma/api/lzma/hardware.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/hardware.h * \brief Hardware information @@ -23,9 +25,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/index.h b/src/liblzma/api/lzma/index.h index 6eee4d6..b17025e 100644 --- a/src/liblzma/api/lzma/index.h +++ b/src/liblzma/api/lzma/index.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/index.h * \brief Handling of .xz Index and related information @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -234,7 +233,7 @@ typedef struct { } block; /** - * \private Internal struct. + * \private Internal data * * Internal data which is used to store the state of the iterator. * The exact format may vary between liblzma versions, so don't @@ -302,6 +301,28 @@ typedef enum { /** + * \brief Mask for return value from lzma_index_checks() for check none + * + * \note This and the other CHECK_MASK macros were added in 5.5.1alpha. + */ +#define LZMA_INDEX_CHECK_MASK_NONE (UINT32_C(1) << LZMA_CHECK_NONE) + +/** + * \brief Mask for return value from lzma_index_checks() for check CRC32 + */ +#define LZMA_INDEX_CHECK_MASK_CRC32 (UINT32_C(1) << LZMA_CHECK_CRC32) + +/** + * \brief Mask for return value from lzma_index_checks() for check CRC64 + */ +#define LZMA_INDEX_CHECK_MASK_CRC64 (UINT32_C(1) << LZMA_CHECK_CRC64) + +/** + * \brief Mask for return value from lzma_index_checks() for check SHA256 + */ +#define LZMA_INDEX_CHECK_MASK_SHA256 (UINT32_C(1) << LZMA_CHECK_SHA256) + +/** * \brief Calculate memory usage of lzma_index * * On disk, the size of the Index field depends on both the number of Records @@ -431,6 +452,7 @@ extern LZMA_API(lzma_ret) lzma_index_stream_flags( * showing the Check types to the user. * * The bitmask is 1 << check_id, e.g. CRC32 is 1 << 1 and SHA-256 is 1 << 10. + * These masks are defined for convenience as LZMA_INDEX_CHECK_MASK_XXX * * \param i Pointer to lzma_index structure * @@ -651,7 +673,7 @@ extern LZMA_API(lzma_bool) lzma_index_iter_locate( * function succeeds, the memory allocated for src * is freed or moved to be part of dest, and all * iterators pointing to src will become invalid. -* \param allocator lzma_allocator for custom allocator functions. + * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * * \return Possible lzma_ret values: @@ -686,7 +708,7 @@ extern LZMA_API(lzma_index *) lzma_index_dup( * \param strm Pointer to properly prepared lzma_stream * \param i Pointer to lzma_index which should be encoded. * - * The valid `action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. + * The valid 'action' values for lzma_code() are LZMA_RUN and LZMA_FINISH. * It is enough to use only one of them (you can choose freely). * * \return Possible lzma_ret values: @@ -715,7 +737,7 @@ extern LZMA_API(lzma_ret) lzma_index_encoder( * don't allow 0 here and return LZMA_PROG_ERROR; * later versions treat 0 as if 1 had been specified. * - * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. * There is no need to use LZMA_FINISH, but it's allowed because it may * simplify certain types of applications. * @@ -771,7 +793,7 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_encode(const lzma_index *i, * lzma_index is allowed to require. The value * pointed by this pointer is modified if and only * if LZMA_MEMLIMIT_ERROR is returned. - * \param allocator lzma_allocator for custom allocator functions. + * \param allocator lzma_allocator for custom allocator functions. * Set to NULL to use malloc() and free(). * \param in Beginning of the input buffer * \param in_pos The next byte will be read from in[*in_pos]. @@ -819,10 +841,10 @@ extern LZMA_API(lzma_ret) lzma_index_buffer_decode(lzma_index **i, * expect to see the same exact value for the same file if you change the * input buffer size or switch to a different liblzma version. * - * Valid `action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. + * Valid 'action' arguments to lzma_code() are LZMA_RUN and LZMA_FINISH. * You only need to use LZMA_RUN; LZMA_FINISH is only supported because it * might be convenient for some applications. If you use LZMA_FINISH and if - * lzma_code() asks the application to seek, remember to reset `action' back + * lzma_code() asks the application to seek, remember to reset 'action' back * to LZMA_RUN unless you hit the end of the file again. * * Possible return values from lzma_code(): diff --git a/src/liblzma/api/lzma/index_hash.h b/src/liblzma/api/lzma/index_hash.h index a2d4c48..68f9024 100644 --- a/src/liblzma/api/lzma/index_hash.h +++ b/src/liblzma/api/lzma/index_hash.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/index_hash.h * \brief Validate Index by using a hash function @@ -9,9 +11,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/lzma12.h b/src/liblzma/api/lzma/lzma12.h index 8ef6ea5..05f5b66 100644 --- a/src/liblzma/api/lzma/lzma12.h +++ b/src/liblzma/api/lzma/lzma12.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/lzma12.h * \brief LZMA1 and LZMA2 filters @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -289,7 +288,7 @@ typedef struct { * \brief Number of literal context bits * * How many of the highest bits of the previous uncompressed - * eight-bit byte (also known as `literal') are taken into + * eight-bit byte (also known as 'literal') are taken into * account when predicting the bits of the next literal. * * E.g. in typical English text, an upper-case letter is diff --git a/src/liblzma/api/lzma/stream_flags.h b/src/liblzma/api/lzma/stream_flags.h index 7622a62..a33fe46 100644 --- a/src/liblzma/api/lzma/stream_flags.h +++ b/src/liblzma/api/lzma/stream_flags.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/stream_flags.h * \brief .xz Stream Header and Stream Footer encoder and decoder @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL diff --git a/src/liblzma/api/lzma/version.h b/src/liblzma/api/lzma/version.h index 8dac382..53526b9 100644 --- a/src/liblzma/api/lzma/version.h +++ b/src/liblzma/api/lzma/version.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/version.h * \brief Version number @@ -6,9 +8,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -20,10 +19,10 @@ #define LZMA_VERSION_MAJOR 5 /** \brief Minor version number of the liblzma release. */ -#define LZMA_VERSION_MINOR 4 +#define LZMA_VERSION_MINOR 6 /** \brief Patch version number of the liblzma release. */ -#define LZMA_VERSION_PATCH 5 +#define LZMA_VERSION_PATCH 2 /** * \brief Version stability marker diff --git a/src/liblzma/api/lzma/vli.h b/src/liblzma/api/lzma/vli.h index f9ad155..6b04902 100644 --- a/src/liblzma/api/lzma/vli.h +++ b/src/liblzma/api/lzma/vli.h @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /** * \file lzma/vli.h * \brief Variable-length integer handling @@ -17,9 +19,6 @@ /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #ifndef LZMA_H_INTERNAL @@ -68,9 +67,8 @@ typedef uint64_t lzma_vli; * This is useful to test that application has given acceptable values * for example in the uncompressed_size and compressed_size variables. * - * \return True if the integer is representable as VLI or if it - * indicates unknown value. False if the integer cannot be - * represented as VLI. + * \return True if the integer is representable as a VLI or if it + * indicates an unknown value. False otherwise. */ #define lzma_vli_is_valid(vli) \ ((vli) <= LZMA_VLI_MAX || (vli) == LZMA_VLI_UNKNOWN) diff --git a/src/liblzma/check/Makefile.inc b/src/liblzma/check/Makefile.inc index dc011a3..04cf1ff 100644 --- a/src/liblzma/check/Makefile.inc +++ b/src/liblzma/check/Makefile.inc @@ -1,9 +1,8 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## + +## Note: There is no check for COND_CHECK_CRC32 because +## currently crc32 is always enabled. EXTRA_DIST += \ check/crc32_tablegen.c \ @@ -12,9 +11,10 @@ EXTRA_DIST += \ liblzma_la_SOURCES += \ check/check.c \ check/check.h \ - check/crc_macros.h + check/crc_common.h \ + check/crc_x86_clmul.h \ + check/crc32_arm64.h -if COND_CHECK_CRC32 if COND_SMALL liblzma_la_SOURCES += check/crc32_small.c else @@ -28,7 +28,6 @@ else liblzma_la_SOURCES += check/crc32_fast.c endif endif -endif if COND_CHECK_CRC64 if COND_SMALL diff --git a/src/liblzma/check/check.c b/src/liblzma/check/check.c index 428ddae..7734ace 100644 --- a/src/liblzma/check/check.c +++ b/src/liblzma/check/check.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file check.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" diff --git a/src/liblzma/check/check.h b/src/liblzma/check/check.h index 8ae95d5..f0eb117 100644 --- a/src/liblzma/check/check.h +++ b/src/liblzma/check/check.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file check.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_CHECK_H diff --git a/src/liblzma/check/crc32_arm64.h b/src/liblzma/check/crc32_arm64.h new file mode 100644 index 0000000..39c1c63 --- /dev/null +++ b/src/liblzma/check/crc32_arm64.h @@ -0,0 +1,122 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc32_arm64.h +/// \brief CRC32 calculation with ARM64 optimization +// +// Authors: Chenxi Mao +// Jia Tan +// Hans Jansen +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_CRC32_ARM64_H +#define LZMA_CRC32_ARM64_H + +// MSVC always has the CRC intrinsics available when building for ARM64 +// there is no need to include any header files. +#ifndef _MSC_VER +# include <arm_acle.h> +#endif + +// If both versions are going to be built, we need runtime detection +// to check if the instructions are supported. +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) +# if defined(HAVE_GETAUXVAL) || defined(HAVE_ELF_AUX_INFO) +# include <sys/auxv.h> +# elif defined(_WIN32) +# include <processthreadsapi.h> +# elif defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME) +# include <sys/sysctl.h> +# endif +#endif + +// Some EDG-based compilers support ARM64 and define __GNUC__ +// (such as Nvidia's nvcc), but do not support function attributes. +// +// NOTE: Build systems check for this too, keep them in sync with this. +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) +# define crc_attr_target __attribute__((__target__("+crc"))) +#else +# define crc_attr_target +#endif + + +crc_attr_target +static uint32_t +crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) +{ + crc = ~crc; + + // Align the input buffer because this was shown to be + // significantly faster than unaligned accesses. + const size_t align_amount = my_min(size, (0U - (uintptr_t)buf) & 7); + + for (const uint8_t *limit = buf + align_amount; buf < limit; ++buf) + crc = __crc32b(crc, *buf); + + size -= align_amount; + + // Process 8 bytes at a time. The end point is determined by + // ignoring the least significant three bits of size to ensure + // we do not process past the bounds of the buffer. This guarantees + // that limit is a multiple of 8 and is strictly less than size. + for (const uint8_t *limit = buf + (size & ~(size_t)7); + buf < limit; buf += 8) + crc = __crc32d(crc, aligned_read64le(buf)); + + // Process the remaining bytes that are not 8 byte aligned. + for (const uint8_t *limit = buf + (size & 7); buf < limit; ++buf) + crc = __crc32b(crc, *buf); + + return ~crc; +} + + +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) +static inline bool +is_arch_extension_supported(void) +{ +#if defined(HAVE_GETAUXVAL) + return (getauxval(AT_HWCAP) & HWCAP_CRC32) != 0; + +#elif defined(HAVE_ELF_AUX_INFO) + unsigned long feature_flags; + + if (elf_aux_info(AT_HWCAP, &feature_flags, sizeof(feature_flags)) != 0) + return false; + + return (feature_flags & HWCAP_CRC32) != 0; + +#elif defined(_WIN32) + return IsProcessorFeaturePresent( + PF_ARM_V8_CRC32_INSTRUCTIONS_AVAILABLE); + +#elif defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME) + int has_crc32 = 0; + size_t size = sizeof(has_crc32); + + // The sysctlbyname() function requires a string identifier for the + // CPU feature it tests. The Apple documentation lists the string + // "hw.optional.armv8_crc32", which can be found here: + // https://developer.apple.com/documentation/kernel/1387446-sysctlbyname/determining_instruction_set_characteristics#3915619 + if (sysctlbyname("hw.optional.armv8_crc32", &has_crc32, + &size, NULL, 0) != 0) + return false; + + return has_crc32; + +#else + // If a runtime detection method cannot be found, then this must + // be a compile time error. The checks in crc_common.h should ensure + // a runtime detection method is always found if this function is + // built. It would be possible to just return false here, but this + // is inefficient for binary size and runtime since only the generic + // method could ever be used. +# error Runtime detection method unavailable. +#endif +} +#endif + +#endif // LZMA_CRC32_ARM64_H diff --git a/src/liblzma/check/crc32_fast.c b/src/liblzma/check/crc32_fast.c index eed7350..16dbb74 100644 --- a/src/liblzma/check/crc32_fast.c +++ b/src/liblzma/check/crc32_fast.c @@ -1,35 +1,40 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32.c /// \brief CRC32 calculation -/// -/// Calculate the CRC32 using the slice-by-eight algorithm. -/// It is explained in this document: -/// http://www.intel.com/technology/comms/perfnet/download/CRC_generators.pdf -/// The code in this file is not the same as in Intel's paper, but -/// the basic principle is identical. -// -// Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Ilya Kurdyukov +// Hans Jansen // /////////////////////////////////////////////////////////////////////////////// #include "check.h" -#include "crc_macros.h" +#include "crc_common.h" + +#if defined(CRC_X86_CLMUL) +# define BUILDING_CRC32_CLMUL +# include "crc_x86_clmul.h" +#elif defined(CRC32_ARM64) +# include "crc32_arm64.h" +#endif -// If you make any changes, do some benchmarking! Seemingly unrelated -// changes can very easily ruin the performance (and very probably is -// very compiler dependent). -extern LZMA_API(uint32_t) -lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) +#ifdef CRC32_GENERIC + +/////////////////// +// Generic CRC32 // +/////////////////// + +static uint32_t +crc32_generic(const uint8_t *buf, size_t size, uint32_t crc) { crc = ~crc; #ifdef WORDS_BIGENDIAN - crc = bswap32(crc); + crc = byteswap32(crc); #endif if (size > 8) { @@ -75,8 +80,125 @@ lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) crc = lzma_crc32_table[0][*buf++ ^ A(crc)] ^ S8(crc); #ifdef WORDS_BIGENDIAN - crc = bswap32(crc); + crc = byteswap32(crc); #endif return ~crc; } +#endif + + +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) + +////////////////////////// +// Function dispatching // +////////////////////////// + +// If both the generic and arch-optimized implementations are built, then +// the function to use is selected at runtime because the system running +// the binary might not have the arch-specific instruction set extension(s) +// available. The dispatch methods in order of priority: +// +// 1. Constructor. This method uses __attribute__((__constructor__)) to +// set crc32_func at load time. This avoids extra computation (and any +// unlikely threading bugs) on the first call to lzma_crc32() to decide +// which implementation should be used. +// +// 2. First Call Resolution. On the very first call to lzma_crc32(), the +// call will be directed to crc32_dispatch() instead. This will set the +// appropriate implementation function and will not be called again. +// This method does not use any kind of locking but is safe because if +// multiple threads run the dispatcher simultaneously then they will all +// set crc32_func to the same value. + +typedef uint32_t (*crc32_func_type)( + const uint8_t *buf, size_t size, uint32_t crc); + +// This resolver is shared between all dispatch methods. +static crc32_func_type +crc32_resolve(void) +{ + return is_arch_extension_supported() + ? &crc32_arch_optimized : &crc32_generic; +} + + +#ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +// Constructor method. +# define CRC32_SET_FUNC_ATTR __attribute__((__constructor__)) +static crc32_func_type crc32_func; +#else +// First Call Resolution method. +# define CRC32_SET_FUNC_ATTR +static uint32_t crc32_dispatch(const uint8_t *buf, size_t size, uint32_t crc); +static crc32_func_type crc32_func = &crc32_dispatch; +#endif + +CRC32_SET_FUNC_ATTR +static void +crc32_set_func(void) +{ + crc32_func = crc32_resolve(); + return; +} + +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR +static uint32_t +crc32_dispatch(const uint8_t *buf, size_t size, uint32_t crc) +{ + // When __attribute__((__constructor__)) isn't supported, set the + // function pointer without any locking. If multiple threads run + // the detection code in parallel, they will all end up setting + // the pointer to the same value. This avoids the use of + // mythread_once() on every call to lzma_crc32() but this likely + // isn't strictly standards compliant. Let's change it if it breaks. + crc32_set_func(); + return crc32_func(buf, size, crc); +} + +#endif +#endif + + +extern LZMA_API(uint32_t) +lzma_crc32(const uint8_t *buf, size_t size, uint32_t crc) +{ +#if defined(CRC32_GENERIC) && defined(CRC32_ARCH_OPTIMIZED) + // On x86-64, if CLMUL is available, it is the best for non-tiny + // inputs, being over twice as fast as the generic slice-by-four + // version. However, for size <= 16 it's different. In the extreme + // case of size == 1 the generic version can be five times faster. + // At size >= 8 the CLMUL starts to become reasonable. It + // varies depending on the alignment of buf too. + // + // The above doesn't include the overhead of mythread_once(). + // At least on x86-64 GNU/Linux, pthread_once() is very fast but + // it still makes lzma_crc32(buf, 1, crc) 50-100 % slower. When + // size reaches 12-16 bytes the overhead becomes negligible. + // + // So using the generic version for size <= 16 may give better + // performance with tiny inputs but if such inputs happen rarely + // it's not so obvious because then the lookup table of the + // generic version may not be in the processor cache. +#ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS + if (size <= 16) + return crc32_generic(buf, size, crc); +#endif + +/* +#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR + // See crc32_dispatch(). This would be the alternative which uses + // locking and doesn't use crc32_dispatch(). Note that on Windows + // this method needs Vista threads. + mythread_once(crc64_set_func); +#endif +*/ + return crc32_func(buf, size, crc); + +#elif defined(CRC32_ARCH_OPTIMIZED) + return crc32_arch_optimized(buf, size, crc); + +#else + return crc32_generic(buf, size, crc); +#endif +} diff --git a/src/liblzma/check/crc32_small.c b/src/liblzma/check/crc32_small.c index 186966e..6a1bd66 100644 --- a/src/liblzma/check/crc32_small.c +++ b/src/liblzma/check/crc32_small.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_small.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" diff --git a/src/liblzma/check/crc32_table.c b/src/liblzma/check/crc32_table.c index b11762a..c141cef 100644 --- a/src/liblzma/check/crc32_table.c +++ b/src/liblzma/check/crc32_table.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_table.c @@ -5,18 +7,36 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" + +// FIXME: Compared to crc_common.h this has to check for __x86_64__ too +// so that in 32-bit builds crc32_x86.S won't break due to a missing table. +#if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ + && defined(__SSE4_1__) && defined(__PCLMUL__)) \ + || (defined(__e2k__) && __iset__ >= 6)) +# define NO_CRC32_TABLE + +#elif defined(HAVE_ARM64_CRC32) \ + && !defined(WORDS_BIGENDIAN) \ + && defined(__ARM_FEATURE_CRC32) +# define NO_CRC32_TABLE +#endif + + +#if !defined(HAVE_ENCODERS) && defined(NO_CRC32_TABLE) +// No table needed. Use a typedef to avoid an empty translation unit. +typedef void lzma_crc32_dummy; + +#else // Having the declaration here silences clang -Wmissing-variable-declarations. extern const uint32_t lzma_crc32_table[8][256]; -#ifdef WORDS_BIGENDIAN -# include "crc32_table_be.h" -#else -# include "crc32_table_le.h" +# ifdef WORDS_BIGENDIAN +# include "crc32_table_be.h" +# else +# include "crc32_table_le.h" +# endif #endif diff --git a/src/liblzma/check/crc32_table_be.h b/src/liblzma/check/crc32_table_be.h index c483cb6..505c230 100644 --- a/src/liblzma/check/crc32_table_be.h +++ b/src/liblzma/check/crc32_table_be.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_crc32_table[8][256] = { { diff --git a/src/liblzma/check/crc32_table_le.h b/src/liblzma/check/crc32_table_le.h index 25f4fc4..e89c21a 100644 --- a/src/liblzma/check/crc32_table_le.h +++ b/src/liblzma/check/crc32_table_le.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_crc32_table[8][256] = { { diff --git a/src/liblzma/check/crc32_tablegen.c b/src/liblzma/check/crc32_tablegen.c index 31a4d27..b8cf459 100644 --- a/src/liblzma/check/crc32_tablegen.c +++ b/src/liblzma/check/crc32_tablegen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc32_tablegen.c @@ -9,9 +11,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <stdio.h> @@ -44,7 +43,7 @@ init_crc32_table(void) #ifdef WORDS_BIGENDIAN for (size_t s = 0; s < 8; ++s) for (size_t b = 0; b < 256; ++b) - crc32_table[s][b] = bswap32(crc32_table[s][b]); + crc32_table[s][b] = byteswap32(crc32_table[s][b]); #endif return; @@ -54,9 +53,11 @@ init_crc32_table(void) static void print_crc32_table(void) { - printf("/* This file has been automatically generated by " - "crc32_tablegen.c. */\n\n" - "const uint32_t lzma_crc32_table[8][256] = {\n\t{"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc32_tablegen.c.\n\n" + "const uint32_t lzma_crc32_table[8][256] = {\n\t{"); for (size_t s = 0; s < 8; ++s) { for (size_t b = 0; b < 256; ++b) { @@ -82,9 +83,11 @@ print_crc32_table(void) static void print_lz_table(void) { - printf("/* This file has been automatically generated by " - "crc32_tablegen.c. */\n\n" - "const uint32_t lzma_lz_hash_table[256] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc32_tablegen.c.\n\n" + "const uint32_t lzma_lz_hash_table[256] = {"); for (size_t b = 0; b < 256; ++b) { if ((b % 4) == 0) diff --git a/src/liblzma/check/crc32_x86.S b/src/liblzma/check/crc32_x86.S index 4f395df..ddc3cee 100644 --- a/src/liblzma/check/crc32_x86.S +++ b/src/liblzma/check/crc32_x86.S @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Speed-optimized CRC32 using slicing-by-eight algorithm * @@ -11,9 +13,6 @@ * Authors: Igor Pavlov (original version) * Lasse Collin (AT&T syntax, PIC support, better portability) * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * * This code needs lzma_crc32_table, which can be created using the * following C code: diff --git a/src/liblzma/check/crc64_fast.c b/src/liblzma/check/crc64_fast.c index 0c8622a..0ce83fe 100644 --- a/src/liblzma/check/crc64_fast.c +++ b/src/liblzma/check/crc64_fast.c @@ -1,85 +1,30 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64.c /// \brief CRC64 calculation -/// -/// There are two methods in this file. crc64_generic uses the -/// the slice-by-four algorithm. This is the same idea that is -/// used in crc32_fast.c, but for CRC64 we use only four tables -/// instead of eight to avoid increasing CPU cache usage. -/// -/// crc64_clmul uses 32/64-bit x86 SSSE3, SSE4.1, and CLMUL instructions. -/// It was derived from -/// https://www.intel.com/content/dam/www/public/us/en/documents/white-papers/fast-crc-computation-generic-polynomials-pclmulqdq-paper.pdf -/// and the public domain code from https://github.com/rawrunprotected/crc -/// (URLs were checked on 2022-11-07). -/// -/// FIXME: Builds for 32-bit x86 use crc64_x86.S by default instead -/// of this file and thus CLMUL version isn't available on 32-bit x86 -/// unless configured with --disable-assembler. Even then the lookup table -/// isn't omitted in crc64_table.c since it doesn't know that assembly -/// code has been disabled. // // Authors: Lasse Collin // Ilya Kurdyukov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" +#include "crc_common.h" -#undef CRC_GENERIC -#undef CRC_CLMUL -#undef CRC_USE_GENERIC_FOR_SMALL_INPUTS - -// If CLMUL cannot be used then only the generic slice-by-four is built. -#if !defined(HAVE_USABLE_CLMUL) -# define CRC_GENERIC 1 - -// If CLMUL is allowed unconditionally in the compiler options then the -// generic version can be omitted. Note that this doesn't work with MSVC -// as I don't know how to detect the features here. -// -// NOTE: Keep this this in sync with crc64_table.c. -#elif (defined(__SSSE3__) && defined(__SSE4_1__) && defined(__PCLMUL__)) \ - || (defined(__e2k__) && __iset__ >= 6) -# define CRC_CLMUL 1 - -// Otherwise build both and detect at runtime which version to use. -#else -# define CRC_GENERIC 1 -# define CRC_CLMUL 1 - -/* - // The generic code is much faster with 1-8-byte inputs and has - // similar performance up to 16 bytes at least in microbenchmarks - // (it depends on input buffer alignment too). If both versions are - // built, this #define will use the generic version for inputs up to - // 16 bytes and CLMUL for bigger inputs. It saves a little in code - // size since the special cases for 0-16-byte inputs will be omitted - // from the CLMUL code. -# define CRC_USE_GENERIC_FOR_SMALL_INPUTS 1 -*/ - -# if defined(_MSC_VER) -# include <intrin.h> -# elif defined(HAVE_CPUID_H) -# include <cpuid.h> -# endif +#if defined(CRC_X86_CLMUL) +# define BUILDING_CRC64_CLMUL +# include "crc_x86_clmul.h" #endif +#ifdef CRC64_GENERIC + ///////////////////////////////// // Generic slice-by-four CRC64 // ///////////////////////////////// -#ifdef CRC_GENERIC - -#include "crc_macros.h" - - #ifdef WORDS_BIGENDIAN # define A1(x) ((x) >> 56) #else @@ -94,7 +39,7 @@ crc64_generic(const uint8_t *buf, size_t size, uint64_t crc) crc = ~crc; #ifdef WORDS_BIGENDIAN - crc = bswap64(crc); + crc = byteswap64(crc); #endif if (size > 4) { @@ -128,7 +73,7 @@ crc64_generic(const uint8_t *buf, size_t size, uint64_t crc) crc = lzma_crc64_table[0][*buf++ ^ A1(crc)] ^ S8(crc); #ifdef WORDS_BIGENDIAN - crc = bswap64(crc); + crc = byteswap64(crc); #endif return ~crc; @@ -136,336 +81,40 @@ crc64_generic(const uint8_t *buf, size_t size, uint64_t crc) #endif -///////////////////// -// x86 CLMUL CRC64 // -///////////////////// - -#ifdef CRC_CLMUL - -#include <immintrin.h> - - -/* -// These functions were used to generate the constants -// at the top of crc64_clmul(). -static uint64_t -calc_lo(uint64_t poly) -{ - uint64_t a = poly; - uint64_t b = 0; - - for (unsigned i = 0; i < 64; ++i) { - b = (b >> 1) | (a << 63); - a = (a >> 1) ^ (a & 1 ? poly : 0); - } - - return b; -} - -static uint64_t -calc_hi(uint64_t poly, uint64_t a) -{ - for (unsigned i = 0; i < 64; ++i) - a = (a >> 1) ^ (a & 1 ? poly : 0); - - return a; -} -*/ - - -#define MASK_L(in, mask, r) \ - r = _mm_shuffle_epi8(in, mask) - -#define MASK_H(in, mask, r) \ - r = _mm_shuffle_epi8(in, _mm_xor_si128(mask, vsign)) - -#define MASK_LH(in, mask, low, high) \ - MASK_L(in, mask, low); \ - MASK_H(in, mask, high) - - -// MSVC (VS2015 - VS2022) produces bad 32-bit x86 code from the CLMUL CRC -// code when optimizations are enabled (release build). According to the bug -// report, the ebx register is corrupted and the calculated result is wrong. -// Trying to workaround the problem with "__asm mov ebx, ebx" didn't help. -// The following pragma works and performance is still good. x86-64 builds -// aren't affected by this problem. -// -// NOTE: Another pragma after the function restores the optimizations. -// If the #if condition here is updated, the other one must be updated too. -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ - && defined(_M_IX86) -# pragma optimize("g", off) -#endif - -// EDG-based compilers (Intel's classic compiler and compiler for E2K) can -// define __GNUC__ but the attribute must not be used with them. -// The new Clang-based ICX needs the attribute. -// -// NOTE: Build systems check for this too, keep them in sync with this. -#if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) -__attribute__((__target__("ssse3,sse4.1,pclmul"))) -#endif -// The intrinsics use 16-byte-aligned reads from buf, thus they may read -// up to 15 bytes before or after the buffer (depending on the alignment -// of the buf argument). The values of the extra bytes are ignored. -// This unavoidably trips -fsanitize=address so address sanitizier has -// to be disabled for this function. -#if lzma_has_attribute(__no_sanitize_address__) -__attribute__((__no_sanitize_address__)) -#endif -static uint64_t -crc64_clmul(const uint8_t *buf, size_t size, uint64_t crc) -{ - // The prototypes of the intrinsics use signed types while most of - // the values are treated as unsigned here. These warnings in this - // function have been checked and found to be harmless so silence them. -#if TUKLIB_GNUC_REQ(4, 6) || defined(__clang__) -# pragma GCC diagnostic push -# pragma GCC diagnostic ignored "-Wsign-conversion" -# pragma GCC diagnostic ignored "-Wconversion" -#endif - -#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS - // The code assumes that there is at least one byte of input. - if (size == 0) - return crc; -#endif - - // const uint64_t poly = 0xc96c5795d7870f42; // CRC polynomial - const uint64_t p = 0x92d8af2baf0e1e85; // (poly << 1) | 1 - const uint64_t mu = 0x9c3e466c172963d5; // (calc_lo(poly) << 1) | 1 - const uint64_t k2 = 0xdabe95afc7875f40; // calc_hi(poly, 1) - const uint64_t k1 = 0xe05dd497ca393ae4; // calc_hi(poly, k2) - const __m128i vfold0 = _mm_set_epi64x(p, mu); - const __m128i vfold1 = _mm_set_epi64x(k2, k1); - - // Create a vector with 8-bit values 0 to 15. This is used to - // construct control masks for _mm_blendv_epi8 and _mm_shuffle_epi8. - const __m128i vramp = _mm_setr_epi32( - 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c); - - // This is used to inverse the control mask of _mm_shuffle_epi8 - // so that bytes that wouldn't be picked with the original mask - // will be picked and vice versa. - const __m128i vsign = _mm_set1_epi8(0x80); - - // Memory addresses A to D and the distances between them: - // - // A B C D - // [skip_start][size][skip_end] - // [ size2 ] - // - // A and D are 16-byte aligned. B and C are 1-byte aligned. - // skip_start and skip_end are 0-15 bytes. size is at least 1 byte. - // - // A = aligned_buf will initially point to this address. - // B = The address pointed by the caller-supplied buf. - // C = buf + size == aligned_buf + size2 - // D = buf + size + skip_end == aligned_buf + size2 + skip_end - const size_t skip_start = (size_t)((uintptr_t)buf & 15); - const size_t skip_end = (size_t)((0U - (uintptr_t)(buf + size)) & 15); - const __m128i *aligned_buf = (const __m128i *)( - (uintptr_t)buf & ~(uintptr_t)15); - - // If size2 <= 16 then the whole input fits into a single 16-byte - // vector. If size2 > 16 then at least two 16-byte vectors must - // be processed. If size2 > 16 && size <= 16 then there is only - // one 16-byte vector's worth of input but it is unaligned in memory. - // - // NOTE: There is no integer overflow here if the arguments are valid. - // If this overflowed, buf + size would too. - size_t size2 = skip_start + size; - - // Masks to be used with _mm_blendv_epi8 and _mm_shuffle_epi8: - // The first skip_start or skip_end bytes in the vectors will have - // the high bit (0x80) set. _mm_blendv_epi8 and _mm_shuffle_epi8 - // will produce zeros for these positions. (Bitwise-xor of these - // masks with vsign will produce the opposite behavior.) - const __m128i mask_start - = _mm_sub_epi8(vramp, _mm_set1_epi8(skip_start)); - const __m128i mask_end = _mm_sub_epi8(vramp, _mm_set1_epi8(skip_end)); - - // Get the first 1-16 bytes into data0. If loading less than 16 bytes, - // the bytes are loaded to the high bits of the vector and the least - // significant positions are filled with zeros. - const __m128i data0 = _mm_blendv_epi8(_mm_load_si128(aligned_buf), - _mm_setzero_si128(), mask_start); - ++aligned_buf; - -#if defined(__i386__) || defined(_M_IX86) - const __m128i initial_crc = _mm_set_epi64x(0, ~crc); -#else - // GCC and Clang would produce good code with _mm_set_epi64x - // but MSVC needs _mm_cvtsi64_si128 on x86-64. - const __m128i initial_crc = _mm_cvtsi64_si128(~crc); -#endif +#if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) - __m128i v0, v1, v2, v3; - -#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS - if (size <= 16) { - // Right-shift initial_crc by 1-16 bytes based on "size" - // and store the result in v1 (high bytes) and v0 (low bytes). - // - // NOTE: The highest 8 bytes of initial_crc are zeros so - // v1 will be filled with zeros if size >= 8. The highest 8 - // bytes of v1 will always become zeros. - // - // [ v1 ][ v0 ] - // [ initial_crc ] size == 1 - // [ initial_crc ] size == 2 - // [ initial_crc ] size == 15 - // [ initial_crc ] size == 16 (all in v0) - const __m128i mask_low = _mm_add_epi8( - vramp, _mm_set1_epi8(size - 16)); - MASK_LH(initial_crc, mask_low, v0, v1); - - if (size2 <= 16) { - // There are 1-16 bytes of input and it is all - // in data0. Copy the input bytes to v3. If there - // are fewer than 16 bytes, the low bytes in v3 - // will be filled with zeros. That is, the input - // bytes are stored to the same position as - // (part of) initial_crc is in v0. - MASK_L(data0, mask_end, v3); - } else { - // There are 2-16 bytes of input but not all bytes - // are in data0. - const __m128i data1 = _mm_load_si128(aligned_buf); - - // Collect the 2-16 input bytes from data0 and data1 - // to v2 and v3, and bitwise-xor them with the - // low bits of initial_crc in v0. Note that the - // the second xor is below this else-block as it - // is shared with the other branch. - MASK_H(data0, mask_end, v2); - MASK_L(data1, mask_end, v3); - v0 = _mm_xor_si128(v0, v2); - } +////////////////////////// +// Function dispatching // +////////////////////////// - v0 = _mm_xor_si128(v0, v3); - v1 = _mm_alignr_epi8(v1, v0, 8); - } else -#endif - { - const __m128i data1 = _mm_load_si128(aligned_buf); - MASK_LH(initial_crc, mask_start, v0, v1); - v0 = _mm_xor_si128(v0, data0); - v1 = _mm_xor_si128(v1, data1); - -#define FOLD \ - v1 = _mm_xor_si128(v1, _mm_clmulepi64_si128(v0, vfold1, 0x00)); \ - v0 = _mm_xor_si128(v1, _mm_clmulepi64_si128(v0, vfold1, 0x11)); - - while (size2 > 32) { - ++aligned_buf; - size2 -= 16; - FOLD - v1 = _mm_load_si128(aligned_buf); - } - - if (size2 < 32) { - MASK_H(v0, mask_end, v2); - MASK_L(v0, mask_end, v0); - MASK_L(v1, mask_end, v3); - v1 = _mm_or_si128(v2, v3); - } - - FOLD - v1 = _mm_srli_si128(v0, 8); -#undef FOLD - } +// If both the generic and arch-optimized implementations are usable, then +// the function that is used is selected at runtime. See crc32_fast.c. - v1 = _mm_xor_si128(_mm_clmulepi64_si128(v0, vfold1, 0x10), v1); - v0 = _mm_clmulepi64_si128(v1, vfold0, 0x00); - v2 = _mm_clmulepi64_si128(v0, vfold0, 0x10); - v0 = _mm_xor_si128(_mm_xor_si128(v2, _mm_slli_si128(v0, 8)), v1); +typedef uint64_t (*crc64_func_type)( + const uint8_t *buf, size_t size, uint64_t crc); -#if defined(__i386__) || defined(_M_IX86) - return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) | - (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2)); -#else - return ~(uint64_t)_mm_extract_epi64(v0, 1); -#endif - -#if TUKLIB_GNUC_REQ(4, 6) || defined(__clang__) -# pragma GCC diagnostic pop -#endif -} -#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ - && defined(_M_IX86) -# pragma optimize("", on) -#endif -#endif - - -//////////////////////// -// Detect CPU support // -//////////////////////// - -#if defined(CRC_GENERIC) && defined(CRC_CLMUL) -static inline bool -is_clmul_supported(void) +static crc64_func_type +crc64_resolve(void) { - int success = 1; - uint32_t r[4]; // eax, ebx, ecx, edx - -#if defined(_MSC_VER) - // This needs <intrin.h> with MSVC. ICC has it as a built-in - // on all platforms. - __cpuid(r, 1); -#elif defined(HAVE_CPUID_H) - // Compared to just using __asm__ to run CPUID, this also checks - // that CPUID is supported and saves and restores ebx as that is - // needed with GCC < 5 with position-independent code (PIC). - success = __get_cpuid(1, &r[0], &r[1], &r[2], &r[3]); -#else - // Just a fallback that shouldn't be needed. - __asm__("cpuid\n\t" - : "=a"(r[0]), "=b"(r[1]), "=c"(r[2]), "=d"(r[3]) - : "a"(1), "c"(0)); -#endif - - // Returns true if these are supported: - // CLMUL (bit 1 in ecx) - // SSSE3 (bit 9 in ecx) - // SSE4.1 (bit 19 in ecx) - const uint32_t ecx_mask = (1 << 1) | (1 << 9) | (1 << 19); - return success && (r[2] & ecx_mask) == ecx_mask; - - // Alternative methods that weren't used: - // - ICC's _may_i_use_cpu_feature: the other methods should work too. - // - GCC >= 6 / Clang / ICX __builtin_cpu_supports("pclmul") - // - // CPUID decding is needed with MSVC anyway and older GCC. This keeps - // the feature checks in the build system simpler too. The nice thing - // about __builtin_cpu_supports would be that it generates very short - // code as is it only reads a variable set at startup but a few bytes - // doesn't matter here. + return is_arch_extension_supported() + ? &crc64_arch_optimized : &crc64_generic; } - #ifdef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR -# define CRC64_FUNC_INIT # define CRC64_SET_FUNC_ATTR __attribute__((__constructor__)) +static crc64_func_type crc64_func; #else -# define CRC64_FUNC_INIT = &crc64_dispatch # define CRC64_SET_FUNC_ATTR static uint64_t crc64_dispatch(const uint8_t *buf, size_t size, uint64_t crc); +static crc64_func_type crc64_func = &crc64_dispatch; #endif -// Pointer to the the selected CRC64 method. -static uint64_t (*crc64_func)(const uint8_t *buf, size_t size, uint64_t crc) - CRC64_FUNC_INIT; - - CRC64_SET_FUNC_ATTR static void crc64_set_func(void) { - crc64_func = is_clmul_supported() ? &crc64_clmul : &crc64_generic; + crc64_func = crc64_resolve(); return; } @@ -474,12 +123,6 @@ crc64_set_func(void) static uint64_t crc64_dispatch(const uint8_t *buf, size_t size, uint64_t crc) { - // When __attribute__((__constructor__)) isn't supported, set the - // function pointer without any locking. If multiple threads run - // the detection code in parallel, they will all end up setting - // the pointer to the same value. This avoids the use of - // mythread_once() on every call to lzma_crc64() but this likely - // isn't strictly standards compliant. Let's change it if it breaks. crc64_set_func(); return crc64_func(buf, size, crc); } @@ -490,47 +133,22 @@ crc64_dispatch(const uint8_t *buf, size_t size, uint64_t crc) extern LZMA_API(uint64_t) lzma_crc64(const uint8_t *buf, size_t size, uint64_t crc) { -#if defined(CRC_GENERIC) && defined(CRC_CLMUL) - // If CLMUL is available, it is the best for non-tiny inputs, - // being over twice as fast as the generic slice-by-four version. - // However, for size <= 16 it's different. In the extreme case - // of size == 1 the generic version can be five times faster. - // At size >= 8 the CLMUL starts to become reasonable. It - // varies depending on the alignment of buf too. - // - // The above doesn't include the overhead of mythread_once(). - // At least on x86-64 GNU/Linux, pthread_once() is very fast but - // it still makes lzma_crc64(buf, 1, crc) 50-100 % slower. When - // size reaches 12-16 bytes the overhead becomes negligible. - // - // So using the generic version for size <= 16 may give better - // performance with tiny inputs but if such inputs happen rarely - // it's not so obvious because then the lookup table of the - // generic version may not be in the processor cache. +#if defined(CRC64_GENERIC) && defined(CRC64_ARCH_OPTIMIZED) + #ifdef CRC_USE_GENERIC_FOR_SMALL_INPUTS if (size <= 16) return crc64_generic(buf, size, crc); #endif - -/* -#ifndef HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR - // See crc64_dispatch(). This would be the alternative which uses - // locking and doesn't use crc64_dispatch(). Note that on Windows - // this method needs Vista threads. - mythread_once(crc64_set_func); -#endif -*/ - return crc64_func(buf, size, crc); -#elif defined(CRC_CLMUL) - // If CLMUL is used unconditionally without runtime CPU detection - // then omitting the generic version and its 8 KiB lookup table - // makes the library smaller. +#elif defined(CRC64_ARCH_OPTIMIZED) + // If arch-optimized version is used unconditionally without runtime + // CPU detection then omitting the generic version and its 8 KiB + // lookup table makes the library smaller. // // FIXME: Lookup table isn't currently omitted on 32-bit x86, // see crc64_table.c. - return crc64_clmul(buf, size, crc); + return crc64_arch_optimized(buf, size, crc); #else return crc64_generic(buf, size, crc); diff --git a/src/liblzma/check/crc64_small.c b/src/liblzma/check/crc64_small.c index 420f7cf..ee4ea26 100644 --- a/src/liblzma/check/crc64_small.c +++ b/src/liblzma/check/crc64_small.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_small.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" diff --git a/src/liblzma/check/crc64_table.c b/src/liblzma/check/crc64_table.c index 688e527..78e4275 100644 --- a/src/liblzma/check/crc64_table.c +++ b/src/liblzma/check/crc64_table.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_table.c @@ -5,19 +7,21 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" -// FIXME: Compared to crc64_fast.c this has to check for __x86_64__ too +// FIXME: Compared to crc_common.h this has to check for __x86_64__ too // so that in 32-bit builds crc64_x86.S won't break due to a missing table. -#if (defined(__x86_64__) && defined(__SSSE3__) \ +#if defined(HAVE_USABLE_CLMUL) && ((defined(__x86_64__) && defined(__SSSE3__) \ && defined(__SSE4_1__) && defined(__PCLMUL__)) \ - || (defined(__e2k__) && __iset__ >= 6) + || (defined(__e2k__) && __iset__ >= 6)) +# define NO_CRC64_TABLE +#endif + + +#ifdef NO_CRC64_TABLE // No table needed. Use a typedef to avoid an empty translation unit. typedef void lzma_crc64_dummy; diff --git a/src/liblzma/check/crc64_table_be.h b/src/liblzma/check/crc64_table_be.h index ea074f3..db76cc7 100644 --- a/src/liblzma/check/crc64_table_be.h +++ b/src/liblzma/check/crc64_table_be.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc64_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc64_tablegen.c. const uint64_t lzma_crc64_table[4][256] = { { diff --git a/src/liblzma/check/crc64_table_le.h b/src/liblzma/check/crc64_table_le.h index 1196b31..e40a8c8 100644 --- a/src/liblzma/check/crc64_table_le.h +++ b/src/liblzma/check/crc64_table_le.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc64_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc64_tablegen.c. const uint64_t lzma_crc64_table[4][256] = { { diff --git a/src/liblzma/check/crc64_tablegen.c b/src/liblzma/check/crc64_tablegen.c index fddaa7e..2035127 100644 --- a/src/liblzma/check/crc64_tablegen.c +++ b/src/liblzma/check/crc64_tablegen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file crc64_tablegen.c @@ -8,9 +10,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <stdio.h> @@ -43,7 +42,7 @@ init_crc64_table(void) #ifdef WORDS_BIGENDIAN for (size_t s = 0; s < 4; ++s) for (size_t b = 0; b < 256; ++b) - crc64_table[s][b] = bswap64(crc64_table[s][b]); + crc64_table[s][b] = byteswap64(crc64_table[s][b]); #endif return; @@ -53,9 +52,11 @@ init_crc64_table(void) static void print_crc64_table(void) { - printf("/* This file has been automatically generated by " - "crc64_tablegen.c. */\n\n" - "const uint64_t lzma_crc64_table[4][256] = {\n\t{"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by crc64_tablegen.c.\n\n" + "const uint64_t lzma_crc64_table[4][256] = {\n\t{"); for (size_t s = 0; s < 4; ++s) { for (size_t b = 0; b < 256; ++b) { diff --git a/src/liblzma/check/crc64_x86.S b/src/liblzma/check/crc64_x86.S index 9aecf58..47f6081 100644 --- a/src/liblzma/check/crc64_x86.S +++ b/src/liblzma/check/crc64_x86.S @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Speed-optimized CRC64 using slicing-by-four algorithm * @@ -7,9 +9,6 @@ * Authors: Igor Pavlov (original CRC32 assembly code) * Lasse Collin (CRC64 adaptation of the modified CRC32 code) * - * This file has been put into the public domain. - * You can do whatever you want with this file. - * * This code needs lzma_crc64_table, which can be created using the * following C code: diff --git a/src/liblzma/check/crc_common.h b/src/liblzma/check/crc_common.h new file mode 100644 index 0000000..63a7b5c --- /dev/null +++ b/src/liblzma/check/crc_common.h @@ -0,0 +1,137 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc_common.h +/// \brief Some functions and macros for CRC32 and CRC64 +// +// Authors: Lasse Collin +// Ilya Kurdyukov +// Hans Jansen +// Jia Tan +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef LZMA_CRC_COMMON_H +#define LZMA_CRC_COMMON_H + +#include "common.h" + + +#ifdef WORDS_BIGENDIAN +# define A(x) ((x) >> 24) +# define B(x) (((x) >> 16) & 0xFF) +# define C(x) (((x) >> 8) & 0xFF) +# define D(x) ((x) & 0xFF) + +# define S8(x) ((x) << 8) +# define S32(x) ((x) << 32) + +#else +# define A(x) ((x) & 0xFF) +# define B(x) (((x) >> 8) & 0xFF) +# define C(x) (((x) >> 16) & 0xFF) +# define D(x) ((x) >> 24) + +# define S8(x) ((x) >> 8) +# define S32(x) ((x) >> 32) +#endif + + +// CRC CLMUL code needs this because accessing input buffers that aren't +// aligned to the vector size will inherently trip the address sanitizer. +#if lzma_has_attribute(__no_sanitize_address__) +# define crc_attr_no_sanitize_address \ + __attribute__((__no_sanitize_address__)) +#else +# define crc_attr_no_sanitize_address +#endif + +// Keep this in sync with changes to crc32_arm64.h +#if defined(_WIN32) || defined(HAVE_GETAUXVAL) \ + || defined(HAVE_ELF_AUX_INFO) \ + || (defined(__APPLE__) && defined(HAVE_SYSCTLBYNAME)) +# define ARM64_RUNTIME_DETECTION 1 +#endif + + +#undef CRC32_GENERIC +#undef CRC64_GENERIC + +#undef CRC32_ARCH_OPTIMIZED +#undef CRC64_ARCH_OPTIMIZED + +// The x86 CLMUL is used for both CRC32 and CRC64. +#undef CRC_X86_CLMUL + +#undef CRC32_ARM64 +#undef CRC64_ARM64_CLMUL + +#undef CRC_USE_GENERIC_FOR_SMALL_INPUTS + +// ARM64 CRC32 instruction is only useful for CRC32. Currently, only +// little endian is supported since we were unable to test on a big +// endian machine. +// +// NOTE: Keep this and the next check in sync with the macro +// NO_CRC32_TABLE in crc32_table.c +#if defined(HAVE_ARM64_CRC32) && !defined(WORDS_BIGENDIAN) +// Allow ARM64 CRC32 instruction without a runtime check if +// __ARM_FEATURE_CRC32 is defined. GCC and Clang only define this if the +// proper compiler options are used. +# if defined(__ARM_FEATURE_CRC32) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC32_ARM64 1 +# elif defined(ARM64_RUNTIME_DETECTION) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC32_ARM64 1 +# define CRC32_GENERIC 1 +# endif +#endif + +#if defined(HAVE_USABLE_CLMUL) +// If CLMUL is allowed unconditionally in the compiler options then the +// generic version can be omitted. Note that this doesn't work with MSVC +// as I don't know how to detect the features here. +// +// NOTE: Keep this in sync with the NO_CRC32_TABLE macro in crc32_table.c +// and NO_CRC64_TABLE in crc64_table.c. +# if (defined(__SSSE3__) && defined(__SSE4_1__) && defined(__PCLMUL__)) \ + || (defined(__e2k__) && __iset__ >= 6) +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC64_ARCH_OPTIMIZED 1 +# define CRC_X86_CLMUL 1 +# else +# define CRC32_GENERIC 1 +# define CRC64_GENERIC 1 +# define CRC32_ARCH_OPTIMIZED 1 +# define CRC64_ARCH_OPTIMIZED 1 +# define CRC_X86_CLMUL 1 + +/* + // The generic code is much faster with 1-8-byte inputs and + // has similar performance up to 16 bytes at least in + // microbenchmarks (it depends on input buffer alignment + // too). If both versions are built, this #define will use + // the generic version for inputs up to 16 bytes and CLMUL + // for bigger inputs. It saves a little in code size since + // the special cases for 0-16-byte inputs will be omitted + // from the CLMUL code. +# define CRC_USE_GENERIC_FOR_SMALL_INPUTS 1 +*/ +# endif +#endif + +// For CRC32 use the generic slice-by-eight implementation if no optimized +// version is available. +#if !defined(CRC32_ARCH_OPTIMIZED) && !defined(CRC32_GENERIC) +# define CRC32_GENERIC 1 +#endif + +// For CRC64 use the generic slice-by-four implementation if no optimized +// version is available. +#if !defined(CRC64_ARCH_OPTIMIZED) && !defined(CRC64_GENERIC) +# define CRC64_GENERIC 1 +#endif + +#endif diff --git a/src/liblzma/check/crc_macros.h b/src/liblzma/check/crc_macros.h deleted file mode 100644 index a7c21b7..0000000 --- a/src/liblzma/check/crc_macros.h +++ /dev/null @@ -1,30 +0,0 @@ -/////////////////////////////////////////////////////////////////////////////// -// -/// \file crc_macros.h -/// \brief Some endian-dependent macros for CRC32 and CRC64 -// -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. -// -/////////////////////////////////////////////////////////////////////////////// - -#ifdef WORDS_BIGENDIAN -# define A(x) ((x) >> 24) -# define B(x) (((x) >> 16) & 0xFF) -# define C(x) (((x) >> 8) & 0xFF) -# define D(x) ((x) & 0xFF) - -# define S8(x) ((x) << 8) -# define S32(x) ((x) << 32) - -#else -# define A(x) ((x) & 0xFF) -# define B(x) (((x) >> 8) & 0xFF) -# define C(x) (((x) >> 16) & 0xFF) -# define D(x) ((x) >> 24) - -# define S8(x) ((x) >> 8) -# define S32(x) ((x) >> 32) -#endif diff --git a/src/liblzma/check/crc_x86_clmul.h b/src/liblzma/check/crc_x86_clmul.h new file mode 100644 index 0000000..f1254ec --- /dev/null +++ b/src/liblzma/check/crc_x86_clmul.h @@ -0,0 +1,428 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file crc_x86_clmul.h +/// \brief CRC32 and CRC64 implementations using CLMUL instructions. +/// +/// The CRC32 and CRC64 implementations use 32/64-bit x86 SSSE3, SSE4.1, and +/// CLMUL instructions. This is compatible with Elbrus 2000 (E2K) too. +/// +/// They were derived from +/// https://www.researchgate.net/publication/263424619_Fast_CRC_computation +/// and the public domain code from https://github.com/rawrunprotected/crc +/// (URLs were checked on 2023-10-14). +/// +/// While this file has both CRC32 and CRC64 implementations, only one +/// should be built at a time to ensure that crc_simd_body() is inlined +/// even with compilers with which lzma_always_inline expands to plain inline. +/// The version to build is selected by defining BUILDING_CRC32_CLMUL or +/// BUILDING_CRC64_CLMUL before including this file. +/// +/// FIXME: Builds for 32-bit x86 use the assembly .S files by default +/// unless configured with --disable-assembler. Even then the lookup table +/// isn't omitted in crc64_table.c since it doesn't know that assembly +/// code has been disabled. +// +// Authors: Ilya Kurdyukov +// Hans Jansen +// Lasse Collin +// Jia Tan +// +/////////////////////////////////////////////////////////////////////////////// + +// This file must not be included more than once. +#ifdef LZMA_CRC_X86_CLMUL_H +# error crc_x86_clmul.h was included twice. +#endif +#define LZMA_CRC_X86_CLMUL_H + +#include <immintrin.h> + +#if defined(_MSC_VER) +# include <intrin.h> +#elif defined(HAVE_CPUID_H) +# include <cpuid.h> +#endif + + +// EDG-based compilers (Intel's classic compiler and compiler for E2K) can +// define __GNUC__ but the attribute must not be used with them. +// The new Clang-based ICX needs the attribute. +// +// NOTE: Build systems check for this too, keep them in sync with this. +#if (defined(__GNUC__) || defined(__clang__)) && !defined(__EDG__) +# define crc_attr_target \ + __attribute__((__target__("ssse3,sse4.1,pclmul"))) +#else +# define crc_attr_target +#endif + + +#define MASK_L(in, mask, r) r = _mm_shuffle_epi8(in, mask) + +#define MASK_H(in, mask, r) \ + r = _mm_shuffle_epi8(in, _mm_xor_si128(mask, vsign)) + +#define MASK_LH(in, mask, low, high) \ + MASK_L(in, mask, low); \ + MASK_H(in, mask, high) + + +crc_attr_target +crc_attr_no_sanitize_address +static lzma_always_inline void +crc_simd_body(const uint8_t *buf, const size_t size, __m128i *v0, __m128i *v1, + const __m128i vfold16, const __m128i initial_crc) +{ + // Create a vector with 8-bit values 0 to 15. This is used to + // construct control masks for _mm_blendv_epi8 and _mm_shuffle_epi8. + const __m128i vramp = _mm_setr_epi32( + 0x03020100, 0x07060504, 0x0b0a0908, 0x0f0e0d0c); + + // This is used to inverse the control mask of _mm_shuffle_epi8 + // so that bytes that wouldn't be picked with the original mask + // will be picked and vice versa. + const __m128i vsign = _mm_set1_epi8(-0x80); + + // Memory addresses A to D and the distances between them: + // + // A B C D + // [skip_start][size][skip_end] + // [ size2 ] + // + // A and D are 16-byte aligned. B and C are 1-byte aligned. + // skip_start and skip_end are 0-15 bytes. size is at least 1 byte. + // + // A = aligned_buf will initially point to this address. + // B = The address pointed by the caller-supplied buf. + // C = buf + size == aligned_buf + size2 + // D = buf + size + skip_end == aligned_buf + size2 + skip_end + const size_t skip_start = (size_t)((uintptr_t)buf & 15); + const size_t skip_end = (size_t)((0U - (uintptr_t)(buf + size)) & 15); + const __m128i *aligned_buf = (const __m128i *)( + (uintptr_t)buf & ~(uintptr_t)15); + + // If size2 <= 16 then the whole input fits into a single 16-byte + // vector. If size2 > 16 then at least two 16-byte vectors must + // be processed. If size2 > 16 && size <= 16 then there is only + // one 16-byte vector's worth of input but it is unaligned in memory. + // + // NOTE: There is no integer overflow here if the arguments + // are valid. If this overflowed, buf + size would too. + const size_t size2 = skip_start + size; + + // Masks to be used with _mm_blendv_epi8 and _mm_shuffle_epi8: + // The first skip_start or skip_end bytes in the vectors will have + // the high bit (0x80) set. _mm_blendv_epi8 and _mm_shuffle_epi8 + // will produce zeros for these positions. (Bitwise-xor of these + // masks with vsign will produce the opposite behavior.) + const __m128i mask_start + = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_start)); + const __m128i mask_end + = _mm_sub_epi8(vramp, _mm_set1_epi8((char)skip_end)); + + // Get the first 1-16 bytes into data0. If loading less than 16 + // bytes, the bytes are loaded to the high bits of the vector and + // the least significant positions are filled with zeros. + const __m128i data0 = _mm_blendv_epi8(_mm_load_si128(aligned_buf), + _mm_setzero_si128(), mask_start); + aligned_buf++; + + __m128i v2, v3; + +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + if (size <= 16) { + // Right-shift initial_crc by 1-16 bytes based on "size" + // and store the result in v1 (high bytes) and v0 (low bytes). + // + // NOTE: The highest 8 bytes of initial_crc are zeros so + // v1 will be filled with zeros if size >= 8. The highest + // 8 bytes of v1 will always become zeros. + // + // [ v1 ][ v0 ] + // [ initial_crc ] size == 1 + // [ initial_crc ] size == 2 + // [ initial_crc ] size == 15 + // [ initial_crc ] size == 16 (all in v0) + const __m128i mask_low = _mm_add_epi8( + vramp, _mm_set1_epi8((char)(size - 16))); + MASK_LH(initial_crc, mask_low, *v0, *v1); + + if (size2 <= 16) { + // There are 1-16 bytes of input and it is all + // in data0. Copy the input bytes to v3. If there + // are fewer than 16 bytes, the low bytes in v3 + // will be filled with zeros. That is, the input + // bytes are stored to the same position as + // (part of) initial_crc is in v0. + MASK_L(data0, mask_end, v3); + } else { + // There are 2-16 bytes of input but not all bytes + // are in data0. + const __m128i data1 = _mm_load_si128(aligned_buf); + + // Collect the 2-16 input bytes from data0 and data1 + // to v2 and v3, and bitwise-xor them with the + // low bits of initial_crc in v0. Note that the + // the second xor is below this else-block as it + // is shared with the other branch. + MASK_H(data0, mask_end, v2); + MASK_L(data1, mask_end, v3); + *v0 = _mm_xor_si128(*v0, v2); + } + + *v0 = _mm_xor_si128(*v0, v3); + *v1 = _mm_alignr_epi8(*v1, *v0, 8); + } else +#endif + { + // There is more than 16 bytes of input. + const __m128i data1 = _mm_load_si128(aligned_buf); + const __m128i *end = (const __m128i*)( + (const char *)aligned_buf - 16 + size2); + aligned_buf++; + + MASK_LH(initial_crc, mask_start, *v0, *v1); + *v0 = _mm_xor_si128(*v0, data0); + *v1 = _mm_xor_si128(*v1, data1); + + while (aligned_buf < end) { + *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x00)); + *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x11)); + *v1 = _mm_load_si128(aligned_buf++); + } + + if (aligned_buf != end) { + MASK_H(*v0, mask_end, v2); + MASK_L(*v0, mask_end, *v0); + MASK_L(*v1, mask_end, v3); + *v1 = _mm_or_si128(v2, v3); + } + + *v1 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x00)); + *v0 = _mm_xor_si128(*v1, _mm_clmulepi64_si128( + *v0, vfold16, 0x11)); + *v1 = _mm_srli_si128(*v0, 8); + } +} + + +///////////////////// +// x86 CLMUL CRC32 // +///////////////////// + +/* +// These functions were used to generate the constants +// at the top of crc32_arch_optimized(). +static uint64_t +calc_lo(uint64_t p, uint64_t a, int n) +{ + uint64_t b = 0; int i; + for (i = 0; i < n; i++) { + b = b >> 1 | (a & 1) << (n - 1); + a = (a >> 1) ^ ((0 - (a & 1)) & p); + } + return b; +} + +// same as ~crc(&a, sizeof(a), ~0) +static uint64_t +calc_hi(uint64_t p, uint64_t a, int n) +{ + int i; + for (i = 0; i < n; i++) + a = (a >> 1) ^ ((0 - (a & 1)) & p); + return a; +} +*/ + +#ifdef BUILDING_CRC32_CLMUL + +crc_attr_target +crc_attr_no_sanitize_address +static uint32_t +crc32_arch_optimized(const uint8_t *buf, size_t size, uint32_t crc) +{ +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + // The code assumes that there is at least one byte of input. + if (size == 0) + return crc; +#endif + + // uint32_t poly = 0xedb88320; + const int64_t p = 0x1db710640; // p << 1 + const int64_t mu = 0x1f7011641; // calc_lo(p, p, 32) << 1 | 1 + const int64_t k5 = 0x163cd6124; // calc_hi(p, p, 32) << 1 + const int64_t k4 = 0x0ccaa009e; // calc_hi(p, p, 64) << 1 + const int64_t k3 = 0x1751997d0; // calc_hi(p, p, 128) << 1 + + const __m128i vfold4 = _mm_set_epi64x(mu, p); + const __m128i vfold8 = _mm_set_epi64x(0, k5); + const __m128i vfold16 = _mm_set_epi64x(k4, k3); + + __m128i v0, v1, v2; + + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_cvtsi32_si128((int32_t)~crc)); + + v1 = _mm_xor_si128( + _mm_clmulepi64_si128(v0, vfold16, 0x10), v1); // xxx0 + v2 = _mm_shuffle_epi32(v1, 0xe7); // 0xx0 + v0 = _mm_slli_epi64(v1, 32); // [0] + v0 = _mm_clmulepi64_si128(v0, vfold8, 0x00); + v0 = _mm_xor_si128(v0, v2); // [1] [2] + v2 = _mm_clmulepi64_si128(v0, vfold4, 0x10); + v2 = _mm_clmulepi64_si128(v2, vfold4, 0x00); + v0 = _mm_xor_si128(v0, v2); // [2] + return ~(uint32_t)_mm_extract_epi32(v0, 2); +} +#endif // BUILDING_CRC32_CLMUL + + +///////////////////// +// x86 CLMUL CRC64 // +///////////////////// + +/* +// These functions were used to generate the constants +// at the top of crc64_arch_optimized(). +static uint64_t +calc_lo(uint64_t poly) +{ + uint64_t a = poly; + uint64_t b = 0; + + for (unsigned i = 0; i < 64; ++i) { + b = (b >> 1) | (a << 63); + a = (a >> 1) ^ (a & 1 ? poly : 0); + } + + return b; +} + +static uint64_t +calc_hi(uint64_t poly, uint64_t a) +{ + for (unsigned i = 0; i < 64; ++i) + a = (a >> 1) ^ (a & 1 ? poly : 0); + + return a; +} +*/ + +#ifdef BUILDING_CRC64_CLMUL + +// MSVC (VS2015 - VS2022) produces bad 32-bit x86 code from the CLMUL CRC +// code when optimizations are enabled (release build). According to the bug +// report, the ebx register is corrupted and the calculated result is wrong. +// Trying to workaround the problem with "__asm mov ebx, ebx" didn't help. +// The following pragma works and performance is still good. x86-64 builds +// and CRC32 CLMUL aren't affected by this problem. The problem does not +// happen in crc_simd_body() either (which is shared with CRC32 CLMUL anyway). +// +// NOTE: Another pragma after crc64_arch_optimized() restores +// the optimizations. If the #if condition here is updated, +// the other one must be updated too. +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ + && defined(_M_IX86) +# pragma optimize("g", off) +#endif + +crc_attr_target +crc_attr_no_sanitize_address +static uint64_t +crc64_arch_optimized(const uint8_t *buf, size_t size, uint64_t crc) +{ +#ifndef CRC_USE_GENERIC_FOR_SMALL_INPUTS + // The code assumes that there is at least one byte of input. + if (size == 0) + return crc; +#endif + + // const uint64_t poly = 0xc96c5795d7870f42; // CRC polynomial + const uint64_t p = 0x92d8af2baf0e1e85; // (poly << 1) | 1 + const uint64_t mu = 0x9c3e466c172963d5; // (calc_lo(poly) << 1) | 1 + const uint64_t k2 = 0xdabe95afc7875f40; // calc_hi(poly, 1) + const uint64_t k1 = 0xe05dd497ca393ae4; // calc_hi(poly, k2) + + const __m128i vfold8 = _mm_set_epi64x((int64_t)p, (int64_t)mu); + const __m128i vfold16 = _mm_set_epi64x((int64_t)k2, (int64_t)k1); + + __m128i v0, v1, v2; + +#if defined(__i386__) || defined(_M_IX86) + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_set_epi64x(0, (int64_t)~crc)); +#else + // GCC and Clang would produce good code with _mm_set_epi64x + // but MSVC needs _mm_cvtsi64_si128 on x86-64. + crc_simd_body(buf, size, &v0, &v1, vfold16, + _mm_cvtsi64_si128((int64_t)~crc)); +#endif + + v1 = _mm_xor_si128(_mm_clmulepi64_si128(v0, vfold16, 0x10), v1); + v0 = _mm_clmulepi64_si128(v1, vfold8, 0x00); + v2 = _mm_clmulepi64_si128(v0, vfold8, 0x10); + v0 = _mm_xor_si128(_mm_xor_si128(v1, _mm_slli_si128(v0, 8)), v2); + +#if defined(__i386__) || defined(_M_IX86) + return ~(((uint64_t)(uint32_t)_mm_extract_epi32(v0, 3) << 32) | + (uint64_t)(uint32_t)_mm_extract_epi32(v0, 2)); +#else + return ~(uint64_t)_mm_extract_epi64(v0, 1); +#endif +} + +#if defined(_MSC_VER) && !defined(__INTEL_COMPILER) && !defined(__clang__) \ + && defined(_M_IX86) +# pragma optimize("", on) +#endif + +#endif // BUILDING_CRC64_CLMUL + + +// Inlining this function duplicates the function body in crc32_resolve() and +// crc64_resolve(), but this is acceptable because this is a tiny function. +static inline bool +is_arch_extension_supported(void) +{ + int success = 1; + uint32_t r[4]; // eax, ebx, ecx, edx + +#if defined(_MSC_VER) + // This needs <intrin.h> with MSVC. ICC has it as a built-in + // on all platforms. + __cpuid(r, 1); +#elif defined(HAVE_CPUID_H) + // Compared to just using __asm__ to run CPUID, this also checks + // that CPUID is supported and saves and restores ebx as that is + // needed with GCC < 5 with position-independent code (PIC). + success = __get_cpuid(1, &r[0], &r[1], &r[2], &r[3]); +#else + // Just a fallback that shouldn't be needed. + __asm__("cpuid\n\t" + : "=a"(r[0]), "=b"(r[1]), "=c"(r[2]), "=d"(r[3]) + : "a"(1), "c"(0)); +#endif + + // Returns true if these are supported: + // CLMUL (bit 1 in ecx) + // SSSE3 (bit 9 in ecx) + // SSE4.1 (bit 19 in ecx) + const uint32_t ecx_mask = (1 << 1) | (1 << 9) | (1 << 19); + return success && (r[2] & ecx_mask) == ecx_mask; + + // Alternative methods that weren't used: + // - ICC's _may_i_use_cpu_feature: the other methods should work too. + // - GCC >= 6 / Clang / ICX __builtin_cpu_supports("pclmul") + // + // CPUID decding is needed with MSVC anyway and older GCC. This keeps + // the feature checks in the build system simpler too. The nice thing + // about __builtin_cpu_supports would be that it generates very short + // code as is it only reads a variable set at startup but a few bytes + // doesn't matter here. +} diff --git a/src/liblzma/check/sha256.c b/src/liblzma/check/sha256.c index 6feb342..bd0d280 100644 --- a/src/liblzma/check/sha256.c +++ b/src/liblzma/check/sha256.c @@ -1,24 +1,17 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file sha256.c /// \brief SHA-256 -/// -/// \todo Crypto++ has x86 ASM optimizations. They use SSE so if they -/// are imported to liblzma, SSE instructions need to be used -/// conditionally to keep the code working on older boxes. // -// This code is based on the code found from 7-Zip, which has a modified -// version of the SHA-256 found from Crypto++ <https://www.cryptopp.com/>. -// The code was modified a little to fit into liblzma. +// The C code is based on the public domain SHA-256 code found from +// Crypto++ Library 5.5.1 released in 2007: https://www.cryptopp.com/ +// A few minor tweaks have been made in liblzma. // -// Authors: Kevin Springle -// Wei Dai -// Igor Pavlov +// Authors: Wei Dai // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "check.h" diff --git a/src/liblzma/common/Makefile.inc b/src/liblzma/common/Makefile.inc index f0276a2..51b1aae 100644 --- a/src/liblzma/common/Makefile.inc +++ b/src/liblzma/common/Makefile.inc @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## liblzma_la_SOURCES += \ common/common.c \ diff --git a/src/liblzma/common/alone_decoder.c b/src/liblzma/common/alone_decoder.c index 1dc85ba..78af651 100644 --- a/src/liblzma/common/alone_decoder.c +++ b/src/liblzma/common/alone_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "alone_decoder.h" diff --git a/src/liblzma/common/alone_decoder.h b/src/liblzma/common/alone_decoder.h index dfa031a..61ee24d 100644 --- a/src/liblzma/common/alone_decoder.h +++ b/src/liblzma/common/alone_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_ALONE_DECODER_H diff --git a/src/liblzma/common/alone_encoder.c b/src/liblzma/common/alone_encoder.c index 7d3812f..21b0395 100644 --- a/src/liblzma/common/alone_encoder.c +++ b/src/liblzma/common/alone_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file alone_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/auto_decoder.c b/src/liblzma/common/auto_decoder.c index 2a5c089..fdd520f 100644 --- a/src/liblzma/common/auto_decoder.c +++ b/src/liblzma/common/auto_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file auto_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h" diff --git a/src/liblzma/common/block_buffer_decoder.c b/src/liblzma/common/block_buffer_decoder.c index b0ded90..55566cd 100644 --- a/src/liblzma/common/block_buffer_decoder.c +++ b/src/liblzma/common/block_buffer_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_decoder.h" diff --git a/src/liblzma/common/block_buffer_encoder.c b/src/liblzma/common/block_buffer_encoder.c index fdef02d..df3b90e 100644 --- a/src/liblzma/common/block_buffer_encoder.c +++ b/src/liblzma/common/block_buffer_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_buffer_encoder.h" diff --git a/src/liblzma/common/block_buffer_encoder.h b/src/liblzma/common/block_buffer_encoder.h index 653207f..5274ac4 100644 --- a/src/liblzma/common/block_buffer_encoder.h +++ b/src/liblzma/common/block_buffer_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_buffer_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_BUFFER_ENCODER_H diff --git a/src/liblzma/common/block_decoder.c b/src/liblzma/common/block_decoder.c index be647d4..2e369d3 100644 --- a/src/liblzma/common/block_decoder.c +++ b/src/liblzma/common/block_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_decoder.h" diff --git a/src/liblzma/common/block_decoder.h b/src/liblzma/common/block_decoder.h index 718c5ce..2cbf9ba 100644 --- a/src/liblzma/common/block_decoder.h +++ b/src/liblzma/common/block_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_DECODER_H diff --git a/src/liblzma/common/block_encoder.c b/src/liblzma/common/block_encoder.c index 4a136ef..ce8c1de 100644 --- a/src/liblzma/common/block_encoder.c +++ b/src/liblzma/common/block_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_encoder.h" diff --git a/src/liblzma/common/block_encoder.h b/src/liblzma/common/block_encoder.h index bd97c18..b7dfe9a 100644 --- a/src/liblzma/common/block_encoder.h +++ b/src/liblzma/common/block_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_BLOCK_ENCODER_H diff --git a/src/liblzma/common/block_header_decoder.c b/src/liblzma/common/block_header_decoder.c index c4935dc..f0b2fbe 100644 --- a/src/liblzma/common/block_header_decoder.c +++ b/src/liblzma/common/block_header_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_header_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/block_header_encoder.c b/src/liblzma/common/block_header_encoder.c index 160425d..45e57a2 100644 --- a/src/liblzma/common/block_header_encoder.c +++ b/src/liblzma/common/block_header_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_header_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/block_util.c b/src/liblzma/common/block_util.c index acb3111..191f6d4 100644 --- a/src/liblzma/common/block_util.c +++ b/src/liblzma/common/block_util.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file block_util.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/common.c b/src/liblzma/common/common.c index adb50d7..cc0e06a 100644 --- a/src/liblzma/common/common.c +++ b/src/liblzma/common/common.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/common.h b/src/liblzma/common/common.h index 378923e..20af32f 100644 --- a/src/liblzma/common/common.h +++ b/src/liblzma/common/common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_COMMON_H diff --git a/src/liblzma/common/easy_buffer_encoder.c b/src/liblzma/common/easy_buffer_encoder.c index 48eb56f..da610ce 100644 --- a/src/liblzma/common/easy_buffer_encoder.c +++ b/src/liblzma/common/easy_buffer_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_decoder_memusage.c b/src/liblzma/common/easy_decoder_memusage.c index 20bcd5b..0c76f10 100644 --- a/src/liblzma/common/easy_decoder_memusage.c +++ b/src/liblzma/common/easy_decoder_memusage.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_decoder_memusage.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_encoder.c b/src/liblzma/common/easy_encoder.c index 5cb492d..8dfe296 100644 --- a/src/liblzma/common/easy_encoder.c +++ b/src/liblzma/common/easy_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_encoder_memusage.c b/src/liblzma/common/easy_encoder_memusage.c index e910575..1184ac6 100644 --- a/src/liblzma/common/easy_encoder_memusage.c +++ b/src/liblzma/common/easy_encoder_memusage.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_encoder_memusage.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_preset.c b/src/liblzma/common/easy_preset.c index 2f98598..7908a2b 100644 --- a/src/liblzma/common/easy_preset.c +++ b/src/liblzma/common/easy_preset.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_preset.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "easy_preset.h" diff --git a/src/liblzma/common/easy_preset.h b/src/liblzma/common/easy_preset.h index 382ade8..4ef6d04 100644 --- a/src/liblzma/common/easy_preset.h +++ b/src/liblzma/common/easy_preset.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file easy_preset.h @@ -5,11 +7,11 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// +#ifndef LZMA_EASY_PRESET_H +#define LZMA_EASY_PRESET_H + #include "common.h" @@ -30,3 +32,5 @@ typedef struct { /// Set *easy to the settings given by the preset. Returns true on error, /// false on success. extern bool lzma_easy_preset(lzma_options_easy *easy, uint32_t preset); + +#endif diff --git a/src/liblzma/common/file_info.c b/src/liblzma/common/file_info.c index 799bb02..7c85084 100644 --- a/src/liblzma/common/file_info.c +++ b/src/liblzma/common/file_info.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file file_info.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "index_decoder.h" diff --git a/src/liblzma/common/filter_buffer_decoder.c b/src/liblzma/common/filter_buffer_decoder.c index 6620986..cc0d88c 100644 --- a/src/liblzma/common/filter_buffer_decoder.c +++ b/src/liblzma/common/filter_buffer_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h" @@ -24,7 +23,7 @@ lzma_raw_buffer_decode( || out_pos == NULL || *out_pos > out_size) return LZMA_PROG_ERROR; - // Initialize the decoer. + // Initialize the decoder. lzma_next_coder next = LZMA_NEXT_CODER_INIT; return_if_error(lzma_raw_decoder_init(&next, allocator, filters)); diff --git a/src/liblzma/common/filter_buffer_encoder.c b/src/liblzma/common/filter_buffer_encoder.c index dda18e3..7fb8922 100644 --- a/src/liblzma/common/filter_buffer_encoder.c +++ b/src/liblzma/common/filter_buffer_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" diff --git a/src/liblzma/common/filter_common.c b/src/liblzma/common/filter_common.c index fa0927c..d15d9cc 100644 --- a/src/liblzma/common/filter_common.c +++ b/src/liblzma/common/filter_common.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_common.h" @@ -122,6 +121,15 @@ static const struct { .changes_size = false, }, #endif +#if defined(HAVE_ENCODER_RISCV) || defined(HAVE_DECODER_RISCV) + { + .id = LZMA_FILTER_RISCV, + .options_size = sizeof(lzma_options_bcj), + .non_last_ok = true, + .last_ok = false, + .changes_size = false, + }, +#endif #if defined(HAVE_ENCODER_DELTA) || defined(HAVE_DECODER_DELTA) { .id = LZMA_FILTER_DELTA, @@ -145,7 +153,7 @@ lzma_filters_copy(const lzma_filter *src, lzma_filter *real_dest, return LZMA_PROG_ERROR; // Use a temporary destination so that the real destination - // will never be modied if an error occurs. + // will never be modified if an error occurs. lzma_filter dest[LZMA_FILTERS_MAX + 1]; lzma_ret ret; diff --git a/src/liblzma/common/filter_common.h b/src/liblzma/common/filter_common.h index 2e47bb6..95f9fe2 100644 --- a/src/liblzma/common/filter_common.h +++ b/src/liblzma/common/filter_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_COMMON_H diff --git a/src/liblzma/common/filter_decoder.c b/src/liblzma/common/filter_decoder.c index fa53f5b..cbdeb58 100644 --- a/src/liblzma/common/filter_decoder.c +++ b/src/liblzma/common/filter_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h" @@ -121,6 +120,14 @@ static const lzma_filter_decoder decoders[] = { .props_decode = &lzma_simple_props_decode, }, #endif +#ifdef HAVE_DECODER_RISCV + { + .id = LZMA_FILTER_RISCV, + .init = &lzma_simple_riscv_decoder_init, + .memusage = NULL, + .props_decode = &lzma_simple_props_decode, + }, +#endif #ifdef HAVE_DECODER_DELTA { .id = LZMA_FILTER_DELTA, @@ -143,6 +150,16 @@ decoder_find(lzma_vli id) } +// lzma_filter_coder begins with the same members as lzma_filter_decoder. +// This function is a wrapper with a type that is compatible with the +// typedef of lzma_filter_find in filter_common.h. +static const lzma_filter_coder * +coder_find(lzma_vli id) +{ + return (const lzma_filter_coder *)decoder_find(id); +} + + extern LZMA_API(lzma_bool) lzma_filter_decoder_is_supported(lzma_vli id) { @@ -155,7 +172,7 @@ lzma_raw_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *options) { return lzma_raw_coder_init(next, allocator, - options, (lzma_filter_find)(&decoder_find), false); + options, &coder_find, false); } @@ -174,8 +191,7 @@ lzma_raw_decoder(lzma_stream *strm, const lzma_filter *options) extern LZMA_API(uint64_t) lzma_raw_decoder_memusage(const lzma_filter *filters) { - return lzma_raw_coder_memusage( - (lzma_filter_find)(&decoder_find), filters); + return lzma_raw_coder_memusage(&coder_find, filters); } diff --git a/src/liblzma/common/filter_decoder.h b/src/liblzma/common/filter_decoder.h index 2dac602..e610bc1 100644 --- a/src/liblzma/common/filter_decoder.h +++ b/src/liblzma/common/filter_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_DECODER_H diff --git a/src/liblzma/common/filter_encoder.c b/src/liblzma/common/filter_encoder.c index 46fe8af..bc39444 100644 --- a/src/liblzma/common/filter_encoder.c +++ b/src/liblzma/common/filter_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" @@ -33,7 +32,8 @@ typedef struct { /// Calculates the recommended Uncompressed Size for .xz Blocks to /// which the input data can be split to make multithreaded /// encoding possible. If this is NULL, it is assumed that - /// the encoder is fast enough with single thread. + /// the encoder is fast enough with single thread. If the options + /// are invalid, UINT64_MAX is returned. uint64_t (*block_size)(const void *options); /// Tells the size of the Filter Properties field. If options are @@ -158,6 +158,16 @@ static const lzma_filter_encoder encoders[] = { .props_encode = &lzma_simple_props_encode, }, #endif +#ifdef HAVE_ENCODER_RISCV + { + .id = LZMA_FILTER_RISCV, + .init = &lzma_simple_riscv_encoder_init, + .memusage = NULL, + .block_size = NULL, + .props_size_get = &lzma_simple_props_size, + .props_encode = &lzma_simple_props_encode, + }, +#endif #ifdef HAVE_ENCODER_DELTA { .id = LZMA_FILTER_DELTA, @@ -183,6 +193,16 @@ encoder_find(lzma_vli id) } +// lzma_filter_coder begins with the same members as lzma_filter_encoder. +// This function is a wrapper with a type that is compatible with the +// typedef of lzma_filter_find in filter_common.h. +static const lzma_filter_coder * +coder_find(lzma_vli id) +{ + return (const lzma_filter_coder *)encoder_find(id); +} + + extern LZMA_API(lzma_bool) lzma_filter_encoder_is_supported(lzma_vli id) { @@ -219,18 +239,18 @@ lzma_filters_update(lzma_stream *strm, const lzma_filter *filters) extern lzma_ret lzma_raw_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, - const lzma_filter *options) + const lzma_filter *filters) { return lzma_raw_coder_init(next, allocator, - options, (lzma_filter_find)(&encoder_find), true); + filters, &coder_find, true); } extern LZMA_API(lzma_ret) -lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) +lzma_raw_encoder(lzma_stream *strm, const lzma_filter *filters) { - lzma_next_strm_init(lzma_raw_coder_init, strm, options, - (lzma_filter_find)(&encoder_find), true); + lzma_next_strm_init(lzma_raw_coder_init, strm, filters, + &coder_find, true); strm->internal->supported_actions[LZMA_RUN] = true; strm->internal->supported_actions[LZMA_SYNC_FLUSH] = true; @@ -243,31 +263,33 @@ lzma_raw_encoder(lzma_stream *strm, const lzma_filter *options) extern LZMA_API(uint64_t) lzma_raw_encoder_memusage(const lzma_filter *filters) { - return lzma_raw_coder_memusage( - (lzma_filter_find)(&encoder_find), filters); + return lzma_raw_coder_memusage(&coder_find, filters); } -extern uint64_t +extern LZMA_API(uint64_t) lzma_mt_block_size(const lzma_filter *filters) { + if (filters == NULL) + return UINT64_MAX; + uint64_t max = 0; for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { const lzma_filter_encoder *const fe = encoder_find(filters[i].id); + if (fe == NULL) + return UINT64_MAX; + if (fe->block_size != NULL) { const uint64_t size = fe->block_size(filters[i].options); - if (size == 0) - return 0; - if (size > max) max = size; } } - return max; + return max == 0 ? UINT64_MAX : max; } diff --git a/src/liblzma/common/filter_encoder.h b/src/liblzma/common/filter_encoder.h index f1d5683..88f2daf 100644 --- a/src/liblzma/common/filter_encoder.h +++ b/src/liblzma/common/filter_encoder.h @@ -1,13 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // -/// \file filter_encoder.c +/// \file filter_encoder.h /// \brief Filter ID mapping to filter-specific functions // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FILTER_ENCODER_H @@ -16,10 +15,6 @@ #include "common.h" -// FIXME: Might become a part of the public API. -extern uint64_t lzma_mt_block_size(const lzma_filter *filters); - - extern lzma_ret lzma_raw_encoder_init( lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter *filters); diff --git a/src/liblzma/common/filter_flags_decoder.c b/src/liblzma/common/filter_flags_decoder.c index ddfb085..0f5d204 100644 --- a/src/liblzma/common/filter_flags_decoder.c +++ b/src/liblzma/common/filter_flags_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_flags_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_decoder.h" diff --git a/src/liblzma/common/filter_flags_encoder.c b/src/liblzma/common/filter_flags_encoder.c index b57b9fd..e1d6588 100644 --- a/src/liblzma/common/filter_flags_encoder.c +++ b/src/liblzma/common/filter_flags_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file filter_flags_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" diff --git a/src/liblzma/common/hardware_cputhreads.c b/src/liblzma/common/hardware_cputhreads.c index 5d246d2..4ce852b 100644 --- a/src/liblzma/common/hardware_cputhreads.c +++ b/src/liblzma/common/hardware_cputhreads.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware_cputhreads.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/hardware_physmem.c b/src/liblzma/common/hardware_physmem.c index a2bbbe2..1bc3486 100644 --- a/src/liblzma/common/hardware_physmem.c +++ b/src/liblzma/common/hardware_physmem.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware_physmem.c @@ -5,9 +7,6 @@ // // Author: Jonathan Nieder // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/index.c b/src/liblzma/common/index.c index 8a35f43..6add6a6 100644 --- a/src/liblzma/common/index.c +++ b/src/liblzma/common/index.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/index.h b/src/liblzma/common/index.h index 7b27d70..007e118 100644 --- a/src/liblzma/common/index.h +++ b/src/liblzma/common/index.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index.h @@ -12,9 +14,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_INDEX_H diff --git a/src/liblzma/common/index_decoder.c b/src/liblzma/common/index_decoder.c index 19a31b3..4bcb306 100644 --- a/src/liblzma/common/index_decoder.c +++ b/src/liblzma/common/index_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "index_decoder.h" @@ -306,6 +305,12 @@ lzma_index_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, extern LZMA_API(lzma_ret) lzma_index_decoder(lzma_stream *strm, lzma_index **i, uint64_t memlimit) { + // If i isn't NULL, *i must always be initialized due to + // the wording in the API docs. This way it is initialized + // if we return LZMA_PROG_ERROR due to strm == NULL. + if (i != NULL) + *i = NULL; + lzma_next_strm_init(lzma_index_decoder_init, strm, i, memlimit); strm->internal->supported_actions[LZMA_RUN] = true; @@ -320,6 +325,11 @@ lzma_index_buffer_decode(lzma_index **i, uint64_t *memlimit, const lzma_allocator *allocator, const uint8_t *in, size_t *in_pos, size_t in_size) { + // If i isn't NULL, *i must always be initialized due to + // the wording in the API docs. + if (i != NULL) + *i = NULL; + // Sanity checks if (i == NULL || memlimit == NULL || in == NULL || in_pos == NULL || *in_pos > in_size) diff --git a/src/liblzma/common/index_decoder.h b/src/liblzma/common/index_decoder.h index 3fec483..5351d2f 100644 --- a/src/liblzma/common/index_decoder.h +++ b/src/liblzma/common/index_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_INDEX_DECODER_H diff --git a/src/liblzma/common/index_encoder.c b/src/liblzma/common/index_encoder.c index 204490c..ecc299c 100644 --- a/src/liblzma/common/index_encoder.c +++ b/src/liblzma/common/index_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "index_encoder.h" diff --git a/src/liblzma/common/index_encoder.h b/src/liblzma/common/index_encoder.h index 4d55cd1..29ba110 100644 --- a/src/liblzma/common/index_encoder.h +++ b/src/liblzma/common/index_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_INDEX_ENCODER_H diff --git a/src/liblzma/common/index_hash.c b/src/liblzma/common/index_hash.c index 52c3d65..caa5967 100644 --- a/src/liblzma/common/index_hash.c +++ b/src/liblzma/common/index_hash.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file index_hash.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/lzip_decoder.c b/src/liblzma/common/lzip_decoder.c index 88cc7ff..651a0ae 100644 --- a/src/liblzma/common/lzip_decoder.c +++ b/src/liblzma/common/lzip_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzip_decoder.c @@ -6,9 +8,6 @@ // Author: Michał Górny // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzip_decoder.h" diff --git a/src/liblzma/common/lzip_decoder.h b/src/liblzma/common/lzip_decoder.h index 33a01c3..0e1f7be 100644 --- a/src/liblzma/common/lzip_decoder.h +++ b/src/liblzma/common/lzip_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzip_decoder.h @@ -5,9 +7,6 @@ // // Author: Michał Górny // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZIP_DECODER_H diff --git a/src/liblzma/common/memcmplen.h b/src/liblzma/common/memcmplen.h index 99d9c51..394a485 100644 --- a/src/liblzma/common/memcmplen.h +++ b/src/liblzma/common/memcmplen.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file memcmplen.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_MEMCMPLEN_H @@ -24,7 +23,8 @@ // can use the intrinsics without the header file. #if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ && defined(_MSC_VER) \ - && defined(_M_X64) \ + && (defined(_M_X64) \ + || defined(_M_ARM64) || defined(_M_ARM64EC)) \ && !defined(__INTEL_COMPILER) # include <intrin.h> #endif @@ -57,20 +57,35 @@ lzma_memcmplen(const uint8_t *buf1, const uint8_t *buf2, assert(limit <= UINT32_MAX / 2); #if defined(TUKLIB_FAST_UNALIGNED_ACCESS) \ - && ((TUKLIB_GNUC_REQ(3, 4) && defined(__x86_64__)) \ + && (((TUKLIB_GNUC_REQ(3, 4) || defined(__clang__)) \ + && (defined(__x86_64__) \ + || defined(__aarch64__))) \ || (defined(__INTEL_COMPILER) && defined(__x86_64__)) \ || (defined(__INTEL_COMPILER) && defined(_M_X64)) \ - || (defined(_MSC_VER) && defined(_M_X64))) - // I keep this x86-64 only for now since that's where I know this - // to be a good method. This may be fine on other 64-bit CPUs too. - // On big endian one should use xor instead of subtraction and switch - // to __builtin_clzll(). + || (defined(_MSC_VER) && (defined(_M_X64) \ + || defined(_M_ARM64) || defined(_M_ARM64EC)))) + // This is only for x86-64 and ARM64 for now. This might be fine on + // other 64-bit processors too. On big endian one should use xor + // instead of subtraction and switch to __builtin_clzll(). + // + // Reasons to use subtraction instead of xor: + // + // - On some x86-64 processors (Intel Sandy Bridge to Tiger Lake), + // sub+jz and sub+jnz can be fused but xor+jz or xor+jnz cannot. + // Thus using subtraction has potential to be a tiny amount faster + // since the code checks if the quotient is non-zero. + // + // - Some processors (Intel Pentium 4) used to have more ALU + // resources for add/sub instructions than and/or/xor. + // + // The processor info is based on Agner Fog's microarchitecture.pdf + // version 2023-05-26. https://www.agner.org/optimize/ #define LZMA_MEMCMPLEN_EXTRA 8 while (len < limit) { const uint64_t x = read64ne(buf1 + len) - read64ne(buf2 + len); if (x != 0) { // MSVC or Intel C compiler on Windows -# if (defined(_MSC_VER) || defined(__INTEL_COMPILER)) && defined(_M_X64) +# if defined(_MSC_VER) || defined(__INTEL_COMPILER) unsigned long tmp; _BitScanForward64(&tmp, x); len += (uint32_t)tmp >> 3; diff --git a/src/liblzma/common/microlzma_decoder.c b/src/liblzma/common/microlzma_decoder.c index e473373..882cb2c 100644 --- a/src/liblzma/common/microlzma_decoder.c +++ b/src/liblzma/common/microlzma_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file microlzma_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_decoder.h" diff --git a/src/liblzma/common/microlzma_encoder.c b/src/liblzma/common/microlzma_encoder.c index a787ca2..45ec0b1 100644 --- a/src/liblzma/common/microlzma_encoder.c +++ b/src/liblzma/common/microlzma_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file microlzma_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder.h" diff --git a/src/liblzma/common/outqueue.c b/src/liblzma/common/outqueue.c index 71e8648..eb018eb 100644 --- a/src/liblzma/common/outqueue.c +++ b/src/liblzma/common/outqueue.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file outqueue.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "outqueue.h" diff --git a/src/liblzma/common/outqueue.h b/src/liblzma/common/outqueue.h index 596911e..25f0719 100644 --- a/src/liblzma/common/outqueue.h +++ b/src/liblzma/common/outqueue.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file outqueue.h @@ -5,11 +7,11 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// +#ifndef LZMA_OUTQUEUE_H +#define LZMA_OUTQUEUE_H + #include "common.h" @@ -252,3 +254,5 @@ lzma_outq_outbuf_memusage(size_t buf_size) assert(buf_size <= SIZE_MAX - sizeof(lzma_outbuf)); return sizeof(lzma_outbuf) + buf_size; } + +#endif diff --git a/src/liblzma/common/stream_buffer_decoder.c b/src/liblzma/common/stream_buffer_decoder.c index b9745b5..c4f91fb 100644 --- a/src/liblzma/common/stream_buffer_decoder.c +++ b/src/liblzma/common/stream_buffer_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_buffer_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h" diff --git a/src/liblzma/common/stream_buffer_encoder.c b/src/liblzma/common/stream_buffer_encoder.c index 7315759..04d5869 100644 --- a/src/liblzma/common/stream_buffer_encoder.c +++ b/src/liblzma/common/stream_buffer_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_buffer_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/stream_decoder.c b/src/liblzma/common/stream_decoder.c index 6428381..7f42684 100644 --- a/src/liblzma/common/stream_decoder.c +++ b/src/liblzma/common/stream_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_decoder.h" diff --git a/src/liblzma/common/stream_decoder.h b/src/liblzma/common/stream_decoder.h index c13c6ba..5803715 100644 --- a/src/liblzma/common/stream_decoder.h +++ b/src/liblzma/common/stream_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_STREAM_DECODER_H diff --git a/src/liblzma/common/stream_decoder_mt.c b/src/liblzma/common/stream_decoder_mt.c index 76212b4..244624a 100644 --- a/src/liblzma/common/stream_decoder_mt.c +++ b/src/liblzma/common/stream_decoder_mt.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_decoder_mt.c @@ -6,9 +8,6 @@ // Authors: Sebastian Andrzej Siewior // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/stream_encoder.c b/src/liblzma/common/stream_encoder.c index ee92046..e7e5b3f 100644 --- a/src/liblzma/common/stream_encoder.c +++ b/src/liblzma/common/stream_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "block_encoder.h" diff --git a/src/liblzma/common/stream_encoder_mt.c b/src/liblzma/common/stream_encoder_mt.c index f64de9b..f0fef15 100644 --- a/src/liblzma/common/stream_encoder_mt.c +++ b/src/liblzma/common/stream_encoder_mt.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_encoder_mt.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_encoder.h" @@ -979,20 +978,18 @@ get_options(const lzma_mt *options, lzma_options_easy *opt_easy, *filters = opt_easy->filters; } - // Block size - if (options->block_size > 0) { - if (options->block_size > BLOCK_SIZE_MAX) - return LZMA_OPTIONS_ERROR; - + // If the Block size is not set, determine it from the filter chain. + if (options->block_size > 0) *block_size = options->block_size; - } else { - // Determine the Block size from the filter chain. + else *block_size = lzma_mt_block_size(*filters); - if (*block_size == 0) - return LZMA_OPTIONS_ERROR; - assert(*block_size <= BLOCK_SIZE_MAX); - } + // UINT64_MAX > BLOCK_SIZE_MAX, so the second condition + // should be optimized out by any reasonable compiler. + // The second condition should be there in the unlikely event that + // the macros change and UINT64_MAX < BLOCK_SIZE_MAX. + if (*block_size > BLOCK_SIZE_MAX || *block_size == UINT64_MAX) + return LZMA_OPTIONS_ERROR; // Calculate the maximum amount output that a single output buffer // may need to hold. This is the same as the maximum total size of diff --git a/src/liblzma/common/stream_flags_common.c b/src/liblzma/common/stream_flags_common.c index fbe8eb8..41b8dcb 100644 --- a/src/liblzma/common/stream_flags_common.c +++ b/src/liblzma/common/stream_flags_common.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h" diff --git a/src/liblzma/common/stream_flags_common.h b/src/liblzma/common/stream_flags_common.h index 84e96ba..28729db 100644 --- a/src/liblzma/common/stream_flags_common.h +++ b/src/liblzma/common/stream_flags_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_STREAM_FLAGS_COMMON_H diff --git a/src/liblzma/common/stream_flags_decoder.c b/src/liblzma/common/stream_flags_decoder.c index b8d263b..522c98b 100644 --- a/src/liblzma/common/stream_flags_decoder.c +++ b/src/liblzma/common/stream_flags_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h" diff --git a/src/liblzma/common/stream_flags_encoder.c b/src/liblzma/common/stream_flags_encoder.c index b98ab17..f94b5cd 100644 --- a/src/liblzma/common/stream_flags_encoder.c +++ b/src/liblzma/common/stream_flags_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file stream_flags_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "stream_flags_common.h" diff --git a/src/liblzma/common/string_conversion.c b/src/liblzma/common/string_conversion.c index d2c1e80..c899783 100644 --- a/src/liblzma/common/string_conversion.c +++ b/src/liblzma/common/string_conversion.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file string_conversion.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "filter_common.h" @@ -218,12 +217,14 @@ typedef struct { uint16_t offset; union { + // NVHPC has problems with unions that contain pointers that + // are not the first members, so keep "map" at the top. + const name_value_map *map; + struct { uint32_t min; uint32_t max; } range; - - const name_value_map *map; } u; } option_map; @@ -250,7 +251,9 @@ static const char *parse_options(const char **const str, const char *str_end, || defined(HAVE_ENCODER_IA64) \ || defined(HAVE_DECODER_IA64) \ || defined(HAVE_ENCODER_SPARC) \ - || defined(HAVE_DECODER_SPARC) + || defined(HAVE_DECODER_SPARC) \ + || defined(HAVE_ENCODER_RISCV) \ + || defined(HAVE_DECODER_RISCV) static const option_map bcj_optmap[] = { { .name = "start", @@ -509,6 +512,11 @@ static const struct { &parse_bcj, bcj_optmap, 1, 1, true }, #endif +#if defined(HAVE_ENCODER_RISCV) || defined(HAVE_DECODER_RISCV) + { "riscv", sizeof(lzma_options_bcj), LZMA_FILTER_RISCV, + &parse_bcj, bcj_optmap, 1, 1, true }, +#endif + #if defined(HAVE_ENCODER_POWERPC) || defined(HAVE_DECODER_POWERPC) { "powerpc", sizeof(lzma_options_bcj), LZMA_FILTER_POWERPC, &parse_bcj, bcj_optmap, 1, 1, true }, @@ -994,6 +1002,12 @@ extern LZMA_API(const char *) lzma_str_to_filters(const char *str, int *error_pos, lzma_filter *filters, uint32_t flags, const lzma_allocator *allocator) { + // If error_pos isn't NULL, *error_pos must always be set. + // liblzma <= 5.4.6 and <= 5.6.1 have a bug and don't do this + // when str == NULL or filters == NULL or flags are unsupported. + if (error_pos != NULL) + *error_pos = 0; + if (str == NULL || filters == NULL) return "Unexpected NULL pointer argument(s) " "to lzma_str_to_filters()"; diff --git a/src/liblzma/common/vli_decoder.c b/src/liblzma/common/vli_decoder.c index af2799d..3254ccc 100644 --- a/src/liblzma/common/vli_decoder.c +++ b/src/liblzma/common/vli_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/vli_encoder.c b/src/liblzma/common/vli_encoder.c index f864269..3859006 100644 --- a/src/liblzma/common/vli_encoder.c +++ b/src/liblzma/common/vli_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/common/vli_size.c b/src/liblzma/common/vli_size.c index ec1b4fa..c8cb2ec 100644 --- a/src/liblzma/common/vli_size.c +++ b/src/liblzma/common/vli_size.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file vli_size.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/delta/Makefile.inc b/src/liblzma/delta/Makefile.inc index c7739b4..e5b919e 100644 --- a/src/liblzma/delta/Makefile.inc +++ b/src/liblzma/delta/Makefile.inc @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## liblzma_la_SOURCES += \ delta/delta_common.c \ diff --git a/src/liblzma/delta/delta_common.c b/src/liblzma/delta/delta_common.c index 4768201..5dbe253 100644 --- a/src/liblzma/delta/delta_common.c +++ b/src/liblzma/delta/delta_common.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_common.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_common.h" diff --git a/src/liblzma/delta/delta_common.h b/src/liblzma/delta/delta_common.h index 7e7e1ba..bd09127 100644 --- a/src/liblzma/delta/delta_common.h +++ b/src/liblzma/delta/delta_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_common.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_COMMON_H diff --git a/src/liblzma/delta/delta_decoder.c b/src/liblzma/delta/delta_decoder.c index 77cf65c..9f0d49c 100644 --- a/src/liblzma/delta/delta_decoder.c +++ b/src/liblzma/delta/delta_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_decoder.h" @@ -26,6 +25,11 @@ decode_buffer(lzma_delta_coder *coder, uint8_t *buffer, size_t size) } +// For an unknown reason NVIDIA HPC Compiler needs this pragma +// to produce working code. +#ifdef __NVCOMPILER +# pragma routine novector +#endif static lzma_ret delta_decode(void *coder_ptr, const lzma_allocator *allocator, const uint8_t *restrict in, size_t *restrict in_pos, diff --git a/src/liblzma/delta/delta_decoder.h b/src/liblzma/delta/delta_decoder.h index ad89cc6..e2268ed 100644 --- a/src/liblzma/delta/delta_decoder.h +++ b/src/liblzma/delta/delta_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_DECODER_H diff --git a/src/liblzma/delta/delta_encoder.c b/src/liblzma/delta/delta_encoder.c index 056bf74..ba4a50b 100644 --- a/src/liblzma/delta/delta_encoder.c +++ b/src/liblzma/delta/delta_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "delta_encoder.h" diff --git a/src/liblzma/delta/delta_encoder.h b/src/liblzma/delta/delta_encoder.h index 4ab9847..735f0ed 100644 --- a/src/liblzma/delta/delta_encoder.h +++ b/src/liblzma/delta/delta_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_encoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_ENCODER_H diff --git a/src/liblzma/delta/delta_private.h b/src/liblzma/delta/delta_private.h index 0d6cb38..e54721a 100644 --- a/src/liblzma/delta/delta_private.h +++ b/src/liblzma/delta/delta_private.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file delta_private.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_DELTA_PRIVATE_H diff --git a/src/liblzma/liblzma.pc.in b/src/liblzma/liblzma.pc.in index d077cb7..a432992 100644 --- a/src/liblzma/liblzma.pc.in +++ b/src/liblzma/liblzma.pc.in @@ -1,9 +1,5 @@ -# +# SPDX-License-Identifier: 0BSD # Author: Lasse Collin -# -# This file has been put into the public domain. -# You can do whatever you want with this file. -# prefix=@prefix@ exec_prefix=@exec_prefix@ diff --git a/src/liblzma/liblzma_generic.map b/src/liblzma/liblzma_generic.map index bb82167..f74c154 100644 --- a/src/liblzma/liblzma_generic.map +++ b/src/liblzma/liblzma_generic.map @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + XZ_5.0 { global: lzma_alone_decoder; @@ -119,3 +121,8 @@ global: lzma_str_list_filters; lzma_str_to_filters; } XZ_5.2; + +XZ_5.6.0 { +global: + lzma_mt_block_size; +} XZ_5.4; diff --git a/src/liblzma/liblzma_linux.map b/src/liblzma/liblzma_linux.map index 449f5fd..7e4b25e 100644 --- a/src/liblzma/liblzma_linux.map +++ b/src/liblzma/liblzma_linux.map @@ -1,3 +1,5 @@ +/* SPDX-License-Identifier: 0BSD */ + XZ_5.0 { global: lzma_alone_decoder; @@ -134,3 +136,8 @@ global: lzma_str_list_filters; lzma_str_to_filters; } XZ_5.2; + +XZ_5.6.0 { +global: + lzma_mt_block_size; +} XZ_5.4; diff --git a/src/liblzma/liblzma_w32res.rc b/src/liblzma/liblzma_w32res.rc index d4d8159..e225b19 100644 --- a/src/liblzma/liblzma_w32res.rc +++ b/src/liblzma/liblzma_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_DLL diff --git a/src/liblzma/lz/Makefile.inc b/src/liblzma/lz/Makefile.inc index 75742a8..15235d7 100644 --- a/src/liblzma/lz/Makefile.inc +++ b/src/liblzma/lz/Makefile.inc @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## if COND_ENCODER_LZ liblzma_la_SOURCES += \ diff --git a/src/liblzma/lz/lz_decoder.c b/src/liblzma/lz/lz_decoder.c index 06c95c1..92913f2 100644 --- a/src/liblzma/lz/lz_decoder.c +++ b/src/liblzma/lz/lz_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_decoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// // liblzma supports multiple LZ77-based filters. The LZ part is shared @@ -54,9 +53,10 @@ typedef struct { static void lz_decoder_reset(lzma_coder *coder) { - coder->dict.pos = 0; + coder->dict.pos = 2 * LZ_DICT_REPEAT_MAX; coder->dict.full = 0; - coder->dict.buf[coder->dict.size - 1] = '\0'; + coder->dict.buf[2 * LZ_DICT_REPEAT_MAX - 1] = '\0'; + coder->dict.has_wrapped = false; coder->dict.need_reset = false; return; } @@ -70,8 +70,15 @@ decode_buffer(lzma_coder *coder, { while (true) { // Wrap the dictionary if needed. - if (coder->dict.pos == coder->dict.size) - coder->dict.pos = 0; + if (coder->dict.pos == coder->dict.size) { + // See the comment of #define LZ_DICT_REPEAT_MAX. + coder->dict.pos = LZ_DICT_REPEAT_MAX; + coder->dict.has_wrapped = true; + memcpy(coder->dict.buf, coder->dict.buf + + coder->dict.size + - LZ_DICT_REPEAT_MAX, + LZ_DICT_REPEAT_MAX); + } // Store the current dictionary position. It is needed to know // where to start copying to the out[] buffer. @@ -253,21 +260,31 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, // dictionary to the output buffer, since applications are // recommended to give aligned buffers to liblzma. // + // Reserve 2 * LZ_DICT_REPEAT_MAX bytes of extra space which is + // needed for alloc_size. + // // Avoid integer overflow. - if (lz_options.dict_size > SIZE_MAX - 15) + if (lz_options.dict_size > SIZE_MAX - 15 - 2 * LZ_DICT_REPEAT_MAX) return LZMA_MEM_ERROR; lz_options.dict_size = (lz_options.dict_size + 15) & ~((size_t)(15)); + // Reserve extra space as explained in the comment + // of #define LZ_DICT_REPEAT_MAX. + const size_t alloc_size + = lz_options.dict_size + 2 * LZ_DICT_REPEAT_MAX; + // Allocate and initialize the dictionary. - if (coder->dict.size != lz_options.dict_size) { + if (coder->dict.size != alloc_size) { lzma_free(coder->dict.buf, allocator); - coder->dict.buf - = lzma_alloc(lz_options.dict_size, allocator); + coder->dict.buf = lzma_alloc(alloc_size, allocator); if (coder->dict.buf == NULL) return LZMA_MEM_ERROR; - coder->dict.size = lz_options.dict_size; + // NOTE: Yes, alloc_size, not lz_options.dict_size. The way + // coder->dict.full is updated will take care that we will + // still reject distances larger than lz_options.dict_size. + coder->dict.size = alloc_size; } lz_decoder_reset(next->coder); @@ -280,9 +297,12 @@ lzma_lz_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const size_t copy_size = my_min(lz_options.preset_dict_size, lz_options.dict_size); const size_t offset = lz_options.preset_dict_size - copy_size; - memcpy(coder->dict.buf, lz_options.preset_dict + offset, + memcpy(coder->dict.buf + coder->dict.pos, + lz_options.preset_dict + offset, copy_size); - coder->dict.pos = copy_size; + + // dict.pos isn't zero after lz_decoder_reset(). + coder->dict.pos += copy_size; coder->dict.full = copy_size; } diff --git a/src/liblzma/lz/lz_decoder.h b/src/liblzma/lz/lz_decoder.h index ad80d4d..cb61b6e 100644 --- a/src/liblzma/lz/lz_decoder.h +++ b/src/liblzma/lz/lz_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_DECODER_H @@ -17,10 +16,28 @@ #include "common.h" +/// Maximum length of a match rounded up to a nice power of 2 which is +/// a good size for aligned memcpy(). The allocated dictionary buffer will +/// be 2 * LZ_DICT_REPEAT_MAX bytes larger than the actual dictionary size: +/// +/// (1) Every time the decoder reaches the end of the dictionary buffer, +/// the last LZ_DICT_REPEAT_MAX bytes will be copied to the beginning. +/// This way dict_repeat() will only need to copy from one place, +/// never from both the end and beginning of the buffer. +/// +/// (2) The other LZ_DICT_REPEAT_MAX bytes is kept as a buffer between +/// the oldest byte still in the dictionary and the current write +/// position. This way dict_repeat(dict, dict->size - 1, &len) +/// won't need memmove() as the copying cannot overlap. +/// +/// Note that memcpy() still cannot be used if distance < len. +/// +/// LZMA's longest match length is 273 so pick a multiple of 16 above that. +#define LZ_DICT_REPEAT_MAX 288 + + typedef struct { - /// Pointer to the dictionary buffer. It can be an allocated buffer - /// internal to liblzma, or it can a be a buffer given by the - /// application when in single-call mode (not implemented yet). + /// Pointer to the dictionary buffer. uint8_t *buf; /// Write position in dictionary. The next byte will be written to @@ -35,9 +52,16 @@ typedef struct { /// Write limit size_t limit; - /// Size of the dictionary + /// Allocated size of buf. This is 2 * LZ_DICT_REPEAT_MAX bytes + /// larger than the actual dictionary size. This is enforced by + /// how the value for "full" is set; it can be at most + /// "size - 2 * LZ_DICT_REPEAT_MAX". size_t size; + /// True once the dictionary has become full and the writing position + /// has been wrapped in decode_buffer() in lz_decoder.c. + bool has_wrapped; + /// True when dictionary should be reset before decoding more data. bool need_reset; @@ -103,7 +127,16 @@ static inline uint8_t dict_get(const lzma_dict *const dict, const uint32_t distance) { return dict->buf[dict->pos - distance - 1 - + (distance < dict->pos ? 0 : dict->size)]; + + (distance < dict->pos + ? 0 : dict->size - LZ_DICT_REPEAT_MAX)]; +} + + +/// Optimized version of dict_get(dict, 0) +static inline uint8_t +dict_get0(const lzma_dict *const dict) +{ + return dict->buf[dict->pos - 1]; } @@ -132,68 +165,51 @@ dict_repeat(lzma_dict *dict, uint32_t distance, uint32_t *len) uint32_t left = my_min(dict_avail, *len); *len -= left; + size_t back = dict->pos - distance - 1; + if (distance >= dict->pos) + back += dict->size - LZ_DICT_REPEAT_MAX; + // Repeat a block of data from the history. Because memcpy() is faster // than copying byte by byte in a loop, the copying process gets split - // into three cases. + // into two cases. if (distance < left) { // Source and target areas overlap, thus we can't use // memcpy() nor even memmove() safely. do { - dict->buf[dict->pos] = dict_get(dict, distance); - ++dict->pos; + dict->buf[dict->pos++] = dict->buf[back++]; } while (--left > 0); - - } else if (distance < dict->pos) { - // The easiest and fastest case - memcpy(dict->buf + dict->pos, - dict->buf + dict->pos - distance - 1, - left); - dict->pos += left; - } else { - // The bigger the dictionary, the more rare this - // case occurs. We need to "wrap" the dict, thus - // we might need two memcpy() to copy all the data. - assert(dict->full == dict->size); - const uint32_t copy_pos - = dict->pos - distance - 1 + dict->size; - uint32_t copy_size = dict->size - copy_pos; - - if (copy_size < left) { - memmove(dict->buf + dict->pos, dict->buf + copy_pos, - copy_size); - dict->pos += copy_size; - copy_size = left - copy_size; - memcpy(dict->buf + dict->pos, dict->buf, copy_size); - dict->pos += copy_size; - } else { - memmove(dict->buf + dict->pos, dict->buf + copy_pos, - left); - dict->pos += left; - } + memcpy(dict->buf + dict->pos, dict->buf + back, left); + dict->pos += left; } // Update how full the dictionary is. - if (dict->full < dict->pos) - dict->full = dict->pos; + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; - return unlikely(*len != 0); + return *len != 0; +} + + +static inline void +dict_put(lzma_dict *dict, uint8_t byte) +{ + dict->buf[dict->pos++] = byte; + + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; } /// Puts one byte into the dictionary. Returns true if the dictionary was /// already full and the byte couldn't be added. static inline bool -dict_put(lzma_dict *dict, uint8_t byte) +dict_put_safe(lzma_dict *dict, uint8_t byte) { if (unlikely(dict->pos == dict->limit)) return true; - dict->buf[dict->pos++] = byte; - - if (dict->pos > dict->full) - dict->full = dict->pos; - + dict_put(dict, byte); return false; } @@ -217,8 +233,8 @@ dict_write(lzma_dict *restrict dict, const uint8_t *restrict in, *left -= lzma_bufcpy(in, in_pos, in_size, dict->buf, &dict->pos, dict->limit); - if (dict->pos > dict->full) - dict->full = dict->pos; + if (!dict->has_wrapped) + dict->full = dict->pos - 2 * LZ_DICT_REPEAT_MAX; return; } diff --git a/src/liblzma/lz/lz_encoder.c b/src/liblzma/lz/lz_encoder.c index 5489085..4af23e1 100644 --- a/src/liblzma/lz/lz_encoder.c +++ b/src/liblzma/lz/lz_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" @@ -196,9 +195,7 @@ lz_encoder_prepare(lzma_mf *mf, const lzma_allocator *allocator, // For now, the dictionary size is limited to 1.5 GiB. This may grow // in the future if needed, but it needs a little more work than just // changing this check. - if (lz_options->dict_size < LZMA_DICT_SIZE_MIN - || lz_options->dict_size - > (UINT32_C(1) << 30) + (UINT32_C(1) << 29) + if (!IS_ENC_DICT_SIZE_VALID(lz_options->dict_size) || lz_options->nice_len > lz_options->match_len_max) return true; @@ -549,7 +546,7 @@ lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, lzma_lz_options *lz_options)) { #if defined(HAVE_SMALL) && !defined(HAVE_FUNC_ATTRIBUTE_CONSTRUCTOR) - // We need that the CRC32 table has been initialized. + // The CRC32 table must be initialized. lzma_crc32_init(); #endif @@ -569,6 +566,8 @@ lzma_lz_encoder_init(lzma_next_coder *next, const lzma_allocator *allocator, coder->lz.coder = NULL; coder->lz.code = NULL; coder->lz.end = NULL; + coder->lz.options_update = NULL; + coder->lz.set_out_limit = NULL; // mf.size is initialized to silence Valgrind // when used on optimized binaries (GCC may reorder diff --git a/src/liblzma/lz/lz_encoder.h b/src/liblzma/lz/lz_encoder.h index ffcba02..eb197c6 100644 --- a/src/liblzma/lz/lz_encoder.h +++ b/src/liblzma/lz/lz_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_ENCODER_H @@ -17,6 +16,14 @@ #include "common.h" +// For now, the dictionary size is limited to 1.5 GiB. This may grow +// in the future if needed, but it needs a little more work than just +// changing this check. +#define IS_ENC_DICT_SIZE_VALID(size) \ + ((size) >= LZMA_DICT_SIZE_MIN \ + && (size) <= (UINT32_C(1) << 30) + (UINT32_C(1) << 29)) + + /// A table of these is used by the LZ-based encoder to hold /// the length-distance pairs found by the match finder. typedef struct { @@ -153,9 +160,13 @@ typedef struct { /// Maximum search depth uint32_t depth; - /// TODO: Comment + /// Initial dictionary for the match finder to search. const uint8_t *preset_dict; + /// If the preset dictionary is NULL, this value is ignored. + /// Otherwise this member must indicate the preset dictionary's + /// buffer size. If this size is larger than dict_size, then only + /// the dict_size sized tail of the preset_dict will be used. uint32_t preset_dict_size; } lzma_lz_options; @@ -217,7 +228,7 @@ typedef struct { // 3. The literals and matches are encoded using e.g. LZMA. // // The bytes that have been ran through the match finder, but not encoded yet, -// are called `read ahead'. +// are called 'read ahead'. /// Get how many bytes the match finder hashes in its initial step. diff --git a/src/liblzma/lz/lz_encoder_hash.h b/src/liblzma/lz/lz_encoder_hash.h index 4d9971a..8ace82b 100644 --- a/src/liblzma/lz/lz_encoder_hash.h +++ b/src/liblzma/lz/lz_encoder_hash.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder_hash.h @@ -5,9 +7,6 @@ // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZ_ENCODER_HASH_H diff --git a/src/liblzma/lz/lz_encoder_hash_table.h b/src/liblzma/lz/lz_encoder_hash_table.h index 8c51717..2b3a60e 100644 --- a/src/liblzma/lz/lz_encoder_hash_table.h +++ b/src/liblzma/lz/lz_encoder_hash_table.h @@ -1,4 +1,6 @@ -/* This file has been automatically generated by crc32_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by crc32_tablegen.c. const uint32_t lzma_lz_hash_table[256] = { 0x00000000, 0x77073096, 0xEE0E612C, 0x990951BA, diff --git a/src/liblzma/lz/lz_encoder_mf.c b/src/liblzma/lz/lz_encoder_mf.c index 1fdc2d7..557c261 100644 --- a/src/liblzma/lz/lz_encoder_mf.c +++ b/src/liblzma/lz/lz_encoder_mf.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lz_encoder_mf.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" diff --git a/src/liblzma/lzma/Makefile.inc b/src/liblzma/lzma/Makefile.inc index 25440d8..dca6b76 100644 --- a/src/liblzma/lzma/Makefile.inc +++ b/src/liblzma/lzma/Makefile.inc @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## EXTRA_DIST += lzma/fastpos_tablegen.c diff --git a/src/liblzma/lzma/fastpos.h b/src/liblzma/lzma/fastpos.h index dbeb16f..d3969a7 100644 --- a/src/liblzma/lzma/fastpos.h +++ b/src/liblzma/lzma/fastpos.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file fastpos.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_FASTPOS_H diff --git a/src/liblzma/lzma/fastpos_table.c b/src/liblzma/lzma/fastpos_table.c index 6a3ceac..4e10e37 100644 --- a/src/liblzma/lzma/fastpos_table.c +++ b/src/liblzma/lzma/fastpos_table.c @@ -1,4 +1,6 @@ -/* This file has been automatically generated by fastpos_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by fastpos_tablegen.c. #include "common.h" #include "fastpos.h" diff --git a/src/liblzma/lzma/fastpos_tablegen.c b/src/liblzma/lzma/fastpos_tablegen.c index 57ed150..957ccb7 100644 --- a/src/liblzma/lzma/fastpos_tablegen.c +++ b/src/liblzma/lzma/fastpos_tablegen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file fastpos_tablegen.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <inttypes.h> @@ -35,11 +34,13 @@ main(void) fastpos[c] = slot_fast; } - printf("/* This file has been automatically generated " - "by fastpos_tablegen.c. */\n\n" - "#include \"common.h\"\n" - "#include \"fastpos.h\"\n\n" - "const uint8_t lzma_fastpos[1 << FASTPOS_BITS] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by fastpos_tablegen.c.\n\n" + "#include \"common.h\"\n" + "#include \"fastpos.h\"\n\n" + "const uint8_t lzma_fastpos[1 << FASTPOS_BITS] = {"); for (size_t i = 0; i < (1 << FASTPOS_BITS); ++i) { if (i % 16 == 0) diff --git a/src/liblzma/lzma/lzma2_decoder.c b/src/liblzma/lzma/lzma2_decoder.c index 567df49..37ab253 100644 --- a/src/liblzma/lzma/lzma2_decoder.c +++ b/src/liblzma/lzma/lzma2_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_decoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma2_decoder.h" diff --git a/src/liblzma/lzma/lzma2_decoder.h b/src/liblzma/lzma/lzma2_decoder.h index ef2dcbf..cdd8b46 100644 --- a/src/liblzma/lzma/lzma2_decoder.h +++ b/src/liblzma/lzma/lzma2_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA2_DECODER_H diff --git a/src/liblzma/lzma/lzma2_encoder.c b/src/liblzma/lzma/lzma2_encoder.c index 4b6b231..e20b75b 100644 --- a/src/liblzma/lzma/lzma2_encoder.c +++ b/src/liblzma/lzma/lzma2_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lz_encoder.h" @@ -409,6 +408,9 @@ lzma_lzma2_block_size(const void *options) { const lzma_options_lzma *const opt = options; + if (!IS_ENC_DICT_SIZE_VALID(opt->dict_size)) + return UINT64_MAX; + // Use at least 1 MiB to keep compression ratio better. return my_max((uint64_t)(opt->dict_size) * 3, UINT64_C(1) << 20); } diff --git a/src/liblzma/lzma/lzma2_encoder.h b/src/liblzma/lzma/lzma2_encoder.h index 515f183..29966a6 100644 --- a/src/liblzma/lzma/lzma2_encoder.h +++ b/src/liblzma/lzma/lzma2_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma2_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA2_ENCODER_H diff --git a/src/liblzma/lzma/lzma_common.h b/src/liblzma/lzma/lzma_common.h index 9d040d9..c3c587f 100644 --- a/src/liblzma/lzma/lzma_common.h +++ b/src/liblzma/lzma/lzma_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_common.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_COMMON_H @@ -84,6 +83,20 @@ typedef enum { ? (state) - 3 \ : (state) - 6)) +/// Like update_literal(state) but when it is already known that +/// is_literal_state(state) is true. +#define update_literal_normal(state) \ + state = ((state) <= STATE_SHORTREP_LIT_LIT \ + ? STATE_LIT_LIT \ + : (state) - 3); + +/// Like update_literal(state) but when it is already known that +/// is_literal_state(state) is false. +#define update_literal_matched(state) \ + state = ((state) <= STATE_LIT_SHORTREP \ + ? (state) - 3 \ + : (state) - 6); + /// Indicate that the latest state was a match. #define update_match(state) \ state = ((state) < LIT_STATES ? STATE_LIT_MATCH : STATE_NONLIT_MATCH) @@ -112,31 +125,33 @@ typedef enum { /// /// Match byte is used when the previous LZMA symbol was something else than /// a literal (that is, it was some kind of match). -#define LITERAL_CODER_SIZE 0x300 +#define LITERAL_CODER_SIZE UINT32_C(0x300) /// Maximum number of literal coders #define LITERAL_CODERS_MAX (1 << LZMA_LCLP_MAX) +/// Calculates the literal_mask that literal_subcoder() needs. +#define literal_mask_calc(lc, lp) \ + ((UINT32_C(0x100) << (lp)) - (UINT32_C(0x100) >> (lc))) + /// Locate the literal coder for the next literal byte. The choice depends on /// - the lowest literal_pos_bits bits of the position of the current /// byte; and /// - the highest literal_context_bits bits of the previous byte. -#define literal_subcoder(probs, lc, lp_mask, pos, prev_byte) \ - ((probs)[(((pos) & (lp_mask)) << (lc)) \ - + ((uint32_t)(prev_byte) >> (8U - (lc)))]) +#define literal_subcoder(probs, lc, literal_mask, pos, prev_byte) \ + ((probs) + UINT32_C(3) * \ + (((((pos) << 8) + (prev_byte)) & (literal_mask)) << (lc))) static inline void -literal_init(probability (*probs)[LITERAL_CODER_SIZE], - uint32_t lc, uint32_t lp) +literal_init(probability *probs, uint32_t lc, uint32_t lp) { assert(lc + lp <= LZMA_LCLP_MAX); - const uint32_t coders = 1U << (lc + lp); + const size_t coders = LITERAL_CODER_SIZE << (lc + lp); - for (uint32_t i = 0; i < coders; ++i) - for (uint32_t j = 0; j < LITERAL_CODER_SIZE; ++j) - bit_reset(probs[i][j]); + for (size_t i = 0; i < coders; ++i) + bit_reset(probs[i]); return; } diff --git a/src/liblzma/lzma/lzma_decoder.c b/src/liblzma/lzma/lzma_decoder.c index 26c148a..0abed02 100644 --- a/src/liblzma/lzma/lzma_decoder.c +++ b/src/liblzma/lzma/lzma_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_decoder.c @@ -5,9 +7,7 @@ /// // Authors: Igor Pavlov // Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -22,25 +22,20 @@ # pragma GCC diagnostic ignored "-Wimplicit-fallthrough" #endif +// Minimum number of input bytes to safely decode one LZMA symbol. +// The worst case is that we decode 22 bits using probabilities and 26 +// direct bits. This may decode at maximum 20 bytes of input. +#define LZMA_IN_REQUIRED 20 -#ifdef HAVE_SMALL // Macros for (somewhat) size-optimized code. -#define seq_4(seq) seq - -#define seq_6(seq) seq - -#define seq_8(seq) seq - -#define seq_len(seq) \ - seq ## _CHOICE, \ - seq ## _CHOICE2, \ - seq ## _BITTREE - +// This is used to decode the match length (how many bytes must be repeated +// from the dictionary). This version is used in the Resumable mode and +// does not unroll any loops. #define len_decode(target, ld, pos_state, seq) \ do { \ case seq ## _CHOICE: \ - rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_if_0_safe(ld.choice, seq ## _CHOICE) { \ rc_update_0(ld.choice); \ probs = ld.low[pos_state];\ limit = LEN_LOW_SYMBOLS; \ @@ -48,7 +43,7 @@ case seq ## _CHOICE: \ } else { \ rc_update_1(ld.choice); \ case seq ## _CHOICE2: \ - rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_if_0_safe(ld.choice2, seq ## _CHOICE2) { \ rc_update_0(ld.choice2); \ probs = ld.mid[pos_state]; \ limit = LEN_MID_SYMBOLS; \ @@ -64,98 +59,39 @@ case seq ## _CHOICE2: \ symbol = 1; \ case seq ## _BITTREE: \ do { \ - rc_bit(probs[symbol], , , seq ## _BITTREE); \ + rc_bit_safe(probs[symbol], , , seq ## _BITTREE); \ } while (symbol < limit); \ target += symbol - limit; \ } while (0) -#else // HAVE_SMALL - -// Unrolled versions -#define seq_4(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3 - -#define seq_6(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3, \ - seq ## 4, \ - seq ## 5 - -#define seq_8(seq) \ - seq ## 0, \ - seq ## 1, \ - seq ## 2, \ - seq ## 3, \ - seq ## 4, \ - seq ## 5, \ - seq ## 6, \ - seq ## 7 - -#define seq_len(seq) \ - seq ## _CHOICE, \ - seq ## _LOW0, \ - seq ## _LOW1, \ - seq ## _LOW2, \ - seq ## _CHOICE2, \ - seq ## _MID0, \ - seq ## _MID1, \ - seq ## _MID2, \ - seq ## _HIGH0, \ - seq ## _HIGH1, \ - seq ## _HIGH2, \ - seq ## _HIGH3, \ - seq ## _HIGH4, \ - seq ## _HIGH5, \ - seq ## _HIGH6, \ - seq ## _HIGH7 -#define len_decode(target, ld, pos_state, seq) \ +// This is the faster version of the match length decoder that does not +// worry about being resumable. It unrolls the bittree decoding loop. +#define len_decode_fast(target, ld, pos_state) \ do { \ symbol = 1; \ -case seq ## _CHOICE: \ - rc_if_0(ld.choice, seq ## _CHOICE) { \ + rc_if_0(ld.choice) { \ rc_update_0(ld.choice); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW0); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW1); \ - rc_bit_case(ld.low[pos_state][symbol], , , seq ## _LOW2); \ - target = symbol - LEN_LOW_SYMBOLS + MATCH_LEN_MIN; \ + rc_bittree3(ld.low[pos_state], \ + -LEN_LOW_SYMBOLS + MATCH_LEN_MIN); \ + target = symbol; \ } else { \ rc_update_1(ld.choice); \ -case seq ## _CHOICE2: \ - rc_if_0(ld.choice2, seq ## _CHOICE2) { \ + rc_if_0(ld.choice2) { \ rc_update_0(ld.choice2); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID0); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID1); \ - rc_bit_case(ld.mid[pos_state][symbol], , , \ - seq ## _MID2); \ - target = symbol - LEN_MID_SYMBOLS \ - + MATCH_LEN_MIN + LEN_LOW_SYMBOLS; \ + rc_bittree3(ld.mid[pos_state], -LEN_MID_SYMBOLS \ + + MATCH_LEN_MIN + LEN_LOW_SYMBOLS); \ + target = symbol; \ } else { \ rc_update_1(ld.choice2); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH0); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH1); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH2); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH3); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH4); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH5); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH6); \ - rc_bit_case(ld.high[symbol], , , seq ## _HIGH7); \ - target = symbol - LEN_HIGH_SYMBOLS \ + rc_bittree8(ld.high, -LEN_HIGH_SYMBOLS \ + MATCH_LEN_MIN \ - + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS; \ + + LEN_LOW_SYMBOLS + LEN_MID_SYMBOLS); \ + target = symbol; \ } \ } \ } while (0) -#endif // HAVE_SMALL - /// Length decoder probabilities; see comments in lzma_common.h. typedef struct { @@ -173,7 +109,7 @@ typedef struct { /////////////////// /// Literals; see comments in lzma_common.h. - probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; + probability literal[LITERAL_CODERS_MAX * LITERAL_CODER_SIZE]; /// If 1, it's a match. Otherwise it's a single 8-bit literal. probability is_match[STATES][POS_STATES_MAX]; @@ -232,7 +168,7 @@ typedef struct { uint32_t pos_mask; // (1U << pb) - 1 uint32_t literal_context_bits; - uint32_t literal_pos_mask; + uint32_t literal_mask; /// Uncompressed size as bytes, or LZMA_VLI_UNKNOWN if end of /// payload marker is expected. @@ -251,22 +187,26 @@ typedef struct { enum { SEQ_NORMALIZE, SEQ_IS_MATCH, - seq_8(SEQ_LITERAL), - seq_8(SEQ_LITERAL_MATCHED), + SEQ_LITERAL, + SEQ_LITERAL_MATCHED, SEQ_LITERAL_WRITE, SEQ_IS_REP, - seq_len(SEQ_MATCH_LEN), - seq_6(SEQ_DIST_SLOT), + SEQ_MATCH_LEN_CHOICE, + SEQ_MATCH_LEN_CHOICE2, + SEQ_MATCH_LEN_BITTREE, + SEQ_DIST_SLOT, SEQ_DIST_MODEL, SEQ_DIRECT, - seq_4(SEQ_ALIGN), + SEQ_ALIGN, SEQ_EOPM, SEQ_IS_REP0, SEQ_SHORTREP, SEQ_IS_REP0_LONG, SEQ_IS_REP1, SEQ_IS_REP2, - seq_len(SEQ_REP_LEN), + SEQ_REP_LEN_CHOICE, + SEQ_REP_LEN_CHOICE2, + SEQ_REP_LEN_BITTREE, SEQ_COPY, } sequence; @@ -321,7 +261,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, const size_t dict_start = dict.pos; // Range decoder - rc_to_local(coder->rc, *in_pos); + rc_to_local(coder->rc, *in_pos, LZMA_IN_REQUIRED); // State uint32_t state = coder->state; @@ -340,7 +280,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, uint32_t offset = coder->offset; uint32_t len = coder->len; - const uint32_t literal_pos_mask = coder->literal_pos_mask; + const uint32_t literal_mask = coder->literal_mask; const uint32_t literal_context_bits = coder->literal_context_bits; // Temporary variables @@ -367,8 +307,24 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, might_finish_without_eopm = true; } - // The main decoder loop. The "switch" is used to restart the decoder at - // correct location. Once restarted, the "switch" is no longer used. + // The main decoder loop. The "switch" is used to resume the decoder at + // correct location. Once resumed, the "switch" is no longer used. + // The decoder loops is split into two modes: + // + // 1 - Non-resumable mode (fast). This is used when it is guaranteed + // there is enough input to decode the next symbol. If the output + // limit is reached, then the decoder loop will save the place + // for the resumable mode to continue. This mode is not used if + // HAVE_SMALL is defined. This is faster than Resumable mode + // because it reduces the number of branches needed and allows + // for more compiler optimizations. + // + // 2 - Resumable mode (slow). This is used when a previous decoder + // loop did not have enough space in the input or output buffers + // to complete. It uses sequence enum values to set remind + // coder->sequence where to resume in the decoder loop. This + // is the only mode used when HAVE_SMALL is defined. + switch (coder->sequence) while (true) { // Calculate new pos_state. This is skipped on the first loop @@ -376,13 +332,339 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, // variables. pos_state = dict.pos & pos_mask; +#ifndef HAVE_SMALL + + /////////////////////////////// + // Non-resumable Mode (fast) // + /////////////////////////////// + + // Go to Resumable mode (1) if there is not enough input to + // safely decode any possible LZMA symbol or (2) if the + // dictionary is full, which may need special checks that + // are only done in the Resumable mode. + if (unlikely(!rc_is_fast_allowed() + || dict.pos == dict.limit)) + goto slow; + + // Decode the first bit from the next LZMA symbol. + // If the bit is a 0, then we handle it as a literal. + // If the bit is a 1, then it is a match of previously + // decoded data. + rc_if_0(coder->is_match[state][pos_state]) { + ///////////////////// + // Decode literal. // + ///////////////////// + + // Update the RC that we have decoded a 0. + rc_update_0(coder->is_match[state][pos_state]); + + // Get the correct probability array from lp and + // lc params. + probs = literal_subcoder(coder->literal, + literal_context_bits, literal_mask, + dict.pos, dict_get0(&dict)); + + if (is_literal_state(state)) { + update_literal_normal(state); + + // Decode literal without match byte. + rc_bittree8(probs, 0); + } else { + update_literal_matched(state); + + // Decode literal with match byte. + rc_matched_literal(probs, + dict_get(&dict, rep0)); + } + + // Write decoded literal to dictionary + dict_put(&dict, symbol); + continue; + } + + /////////////////// + // Decode match. // + /////////////////// + + // Instead of a new byte we are going to decode a + // distance-length pair. The distance represents how far + // back in the dictionary to begin copying. The length + // represents how many bytes to copy. + + rc_update_1(coder->is_match[state][pos_state]); + + rc_if_0(coder->is_rep[state]) { + /////////////////// + // Simple match. // + /////////////////// + + // Not a repeated match. In this case, + // the length (how many bytes to copy) must be + // decoded first. Then, the distance (where to + // start copying) is decoded. + // + // This is also how we know when we are done + // decoding. If the distance decodes to UINT32_MAX, + // then we know to stop decoding (end of payload + // marker). + + rc_update_0(coder->is_rep[state]); + update_match(state); + + // The latest three match distances are kept in + // memory in case there are repeated matches. + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + + // Decode the length of the match. + len_decode_fast(len, coder->match_len_decoder, + pos_state); + + // Next, decode the distance into rep0. + + // The next 6 bits determine how to decode the + // rest of the distance. + probs = coder->dist_slot[get_dist_state(len)]; + + rc_bittree6(probs, -DIST_SLOTS); + assert(symbol <= 63); + + if (symbol < DIST_MODEL_START) { + // If the decoded symbol is < DIST_MODEL_START + // then we use its value directly as the + // match distance. No other bits are needed. + // The only possible distance values + // are [0, 3]. + rep0 = symbol; + } else { + // Use the first two bits of symbol as the + // highest bits of the match distance. + + // "limit" represents the number of low bits + // to decode. + limit = (symbol >> 1) - 1; + assert(limit >= 1 && limit <= 30); + rep0 = 2 + (symbol & 1); + + if (symbol < DIST_MODEL_END) { + // When symbol is > DIST_MODEL_START, + // but symbol < DIST_MODEL_END, then + // it can decode distances between + // [4, 127]. + assert(limit <= 5); + rep0 <<= limit; + assert(rep0 <= 96); + + // -1 is fine, because we start + // decoding at probs[1], not probs[0]. + // NOTE: This violates the C standard, + // since we are doing pointer + // arithmetic past the beginning of + // the array. + assert((int32_t)(rep0 - symbol - 1) + >= -1); + assert((int32_t)(rep0 - symbol - 1) + <= 82); + probs = coder->pos_special + rep0 + - symbol - 1; + symbol = 1; + offset = 1; + + // Variable number (1-5) of bits + // from a reverse bittree. This + // isn't worth manual unrolling. + // + // NOTE: Making one or many of the + // variables (probs, symbol, offset, + // or limit) local here (instead of + // using those declared outside the + // main loop) can affect code size + // and performance which isn't a + // surprise but it's not so clear + // what is the best. + do { + rc_bit_add_if_1(probs, + rep0, offset); + offset <<= 1; + } while (--limit > 0); + } else { + // The distance is >= 128. Decode the + // lower bits without probabilities + // except the lowest four bits. + assert(symbol >= 14); + assert(limit >= 6); + + limit -= ALIGN_BITS; + assert(limit >= 2); + + rc_direct(rep0, limit); + + // Decode the lowest four bits using + // probabilities. + rep0 <<= ALIGN_BITS; + rc_bittree_rev4(coder->pos_align); + rep0 += symbol; + + // If the end of payload marker (EOPM) + // is detected, jump to the safe code. + // The EOPM handling isn't speed + // critical at all. + // + // A final normalization is needed + // after the EOPM (there can be a + // dummy byte to read in some cases). + // If the normalization was done here + // in the fast code, it would need to + // be taken into account in the value + // of LZMA_IN_REQUIRED. Using the + // safe code allows keeping + // LZMA_IN_REQUIRED as 20 instead of + // 21. + if (rep0 == UINT32_MAX) + goto eopm; + } + } + + // Validate the distance we just decoded. + if (unlikely(!dict_is_distance_valid(&dict, rep0))) { + ret = LZMA_DATA_ERROR; + goto out; + } + + } else { + rc_update_1(coder->is_rep[state]); + + ///////////////////// + // Repeated match. // + ///////////////////// + + // The match distance is a value that we have decoded + // recently. The latest four match distances are + // available as rep0, rep1, rep2 and rep3. We will + // now decode which of them is the new distance. + // + // There cannot be a match if we haven't produced + // any output, so check that first. + if (unlikely(!dict_is_distance_valid(&dict, 0))) { + ret = LZMA_DATA_ERROR; + goto out; + } + + rc_if_0(coder->is_rep0[state]) { + rc_update_0(coder->is_rep0[state]); + // The distance is rep0. + + // Decode the next bit to determine if 1 byte + // should be copied from rep0 distance or + // if the number of bytes needs to be decoded. + + // If the next bit is 0, then it is a + // "Short Rep Match" and only 1 bit is copied. + // Otherwise, the length of the match is + // decoded after the "else" statement. + rc_if_0(coder->is_rep0_long[state][pos_state]) { + rc_update_0(coder->is_rep0_long[ + state][pos_state]); + + update_short_rep(state); + dict_put(&dict, dict_get(&dict, rep0)); + continue; + } + + // Repeating more than one byte at + // distance of rep0. + rc_update_1(coder->is_rep0_long[ + state][pos_state]); + + } else { + rc_update_1(coder->is_rep0[state]); + + // The distance is rep1, rep2 or rep3. Once + // we find out which one of these three, it + // is stored to rep0 and rep1, rep2 and rep3 + // are updated accordingly. There is no + // "Short Rep Match" option, so the length + // of the match must always be decoded next. + rc_if_0(coder->is_rep1[state]) { + // The distance is rep1. + rc_update_0(coder->is_rep1[state]); + + const uint32_t distance = rep1; + rep1 = rep0; + rep0 = distance; + + } else { + rc_update_1(coder->is_rep1[state]); + + rc_if_0(coder->is_rep2[state]) { + // The distance is rep2. + rc_update_0(coder->is_rep2[ + state]); + + const uint32_t distance = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; + + } else { + // The distance is rep3. + rc_update_1(coder->is_rep2[ + state]); + + const uint32_t distance = rep3; + rep3 = rep2; + rep2 = rep1; + rep1 = rep0; + rep0 = distance; + } + } + } + + update_long_rep(state); + + // Decode the length of the repeated match. + len_decode_fast(len, coder->rep_len_decoder, + pos_state); + } + + ///////////////////////////////// + // Repeat from history buffer. // + ///////////////////////////////// + + // The length is always between these limits. There is no way + // to trigger the algorithm to set len outside this range. + assert(len >= MATCH_LEN_MIN); + assert(len <= MATCH_LEN_MAX); + + // Repeat len bytes from distance of rep0. + if (unlikely(dict_repeat(&dict, rep0, &len))) { + coder->sequence = SEQ_COPY; + goto out; + } + + continue; + +slow: +#endif + /////////////////////////// + // Resumable Mode (slow) // + /////////////////////////// + + // This is very similar to Non-resumable Mode, so most of the + // comments are not repeated. The main differences are: + // - case labels are used to resume at the correct location. + // - Loops are not unrolled. + // - Range coder macros take an extra sequence argument + // so they can save to coder->sequence the location to + // resume in case there is not enough input. case SEQ_NORMALIZE: case SEQ_IS_MATCH: if (unlikely(might_finish_without_eopm && dict.pos == dict.limit)) { // In rare cases there is a useless byte that needs // to be read anyway. - rc_normalize(SEQ_NORMALIZE); + rc_normalize_safe(SEQ_NORMALIZE); // If the range decoder state is such that we can // be at the end of the LZMA stream, then the @@ -405,49 +687,37 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, eopm_is_valid = true; } - rc_if_0(coder->is_match[state][pos_state], SEQ_IS_MATCH) { - rc_update_0(coder->is_match[state][pos_state]); + rc_if_0_safe(coder->is_match[state][pos_state], SEQ_IS_MATCH) { + ///////////////////// + // Decode literal. // + ///////////////////// - // It's a literal i.e. a single 8-bit byte. + rc_update_0(coder->is_match[state][pos_state]); probs = literal_subcoder(coder->literal, - literal_context_bits, literal_pos_mask, - dict.pos, dict_get(&dict, 0)); + literal_context_bits, literal_mask, + dict.pos, dict_get0(&dict)); symbol = 1; if (is_literal_state(state)) { + update_literal_normal(state); + // Decode literal without match byte. -#ifdef HAVE_SMALL + // The "slow" version does not unroll + // the loop. case SEQ_LITERAL: do { - rc_bit(probs[symbol], , , SEQ_LITERAL); + rc_bit_safe(probs[symbol], , , + SEQ_LITERAL); } while (symbol < (1 << 8)); -#else - rc_bit_case(probs[symbol], , , SEQ_LITERAL0); - rc_bit_case(probs[symbol], , , SEQ_LITERAL1); - rc_bit_case(probs[symbol], , , SEQ_LITERAL2); - rc_bit_case(probs[symbol], , , SEQ_LITERAL3); - rc_bit_case(probs[symbol], , , SEQ_LITERAL4); - rc_bit_case(probs[symbol], , , SEQ_LITERAL5); - rc_bit_case(probs[symbol], , , SEQ_LITERAL6); - rc_bit_case(probs[symbol], , , SEQ_LITERAL7); -#endif } else { + update_literal_matched(state); + // Decode literal with match byte. - // - // We store the byte we compare against - // ("match byte") to "len" to minimize the - // number of variables we need to store - // between decoder calls. len = (uint32_t)(dict_get(&dict, rep0)) << 1; - // The usage of "offset" allows omitting some - // branches, which should give tiny speed - // improvement on some CPUs. "offset" gets - // set to zero if match_bit didn't match. offset = 0x100; -#ifdef HAVE_SMALL case SEQ_LITERAL_MATCHED: do { const uint32_t match_bit @@ -456,7 +726,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, = offset + match_bit + symbol; - rc_bit(probs[subcoder_index], + rc_bit_safe(probs[subcoder_index], offset &= ~match_bit, offset &= match_bit, SEQ_LITERAL_MATCHED); @@ -469,61 +739,10 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, len <<= 1; } while (symbol < (1 << 8)); -#else - // Unroll the loop. - uint32_t match_bit; - uint32_t subcoder_index; - -# define d(seq) \ - case seq: \ - match_bit = len & offset; \ - subcoder_index = offset + match_bit + symbol; \ - rc_bit(probs[subcoder_index], \ - offset &= ~match_bit, \ - offset &= match_bit, \ - seq) - - d(SEQ_LITERAL_MATCHED0); - len <<= 1; - d(SEQ_LITERAL_MATCHED1); - len <<= 1; - d(SEQ_LITERAL_MATCHED2); - len <<= 1; - d(SEQ_LITERAL_MATCHED3); - len <<= 1; - d(SEQ_LITERAL_MATCHED4); - len <<= 1; - d(SEQ_LITERAL_MATCHED5); - len <<= 1; - d(SEQ_LITERAL_MATCHED6); - len <<= 1; - d(SEQ_LITERAL_MATCHED7); -# undef d -#endif } - //update_literal(state); - // Use a lookup table to update to literal state, - // since compared to other state updates, this would - // need two branches. - static const lzma_lzma_state next_state[] = { - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_LIT_LIT, - STATE_MATCH_LIT_LIT, - STATE_REP_LIT_LIT, - STATE_SHORTREP_LIT_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT, - STATE_SHORTREP_LIT, - STATE_MATCH_LIT, - STATE_REP_LIT - }; - state = next_state[state]; - case SEQ_LITERAL_WRITE: - if (unlikely(dict_put(&dict, symbol))) { + if (dict_put_safe(&dict, symbol)) { coder->sequence = SEQ_LITERAL_WRITE; goto out; } @@ -531,64 +750,47 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, continue; } - // Instead of a new byte we are going to get a byte range - // (distance and length) which will be repeated from our - // output history. + /////////////////// + // Decode match. // + /////////////////// rc_update_1(coder->is_match[state][pos_state]); case SEQ_IS_REP: - rc_if_0(coder->is_rep[state], SEQ_IS_REP) { - // Not a repeated match + rc_if_0_safe(coder->is_rep[state], SEQ_IS_REP) { + /////////////////// + // Simple match. // + /////////////////// + rc_update_0(coder->is_rep[state]); update_match(state); - // The latest three match distances are kept in - // memory in case there are repeated matches. rep3 = rep2; rep2 = rep1; rep1 = rep0; - // Decode the length of the match. len_decode(len, coder->match_len_decoder, pos_state, SEQ_MATCH_LEN); - // Prepare to decode the highest two bits of the - // match distance. probs = coder->dist_slot[get_dist_state(len)]; symbol = 1; -#ifdef HAVE_SMALL case SEQ_DIST_SLOT: do { - rc_bit(probs[symbol], , , SEQ_DIST_SLOT); + rc_bit_safe(probs[symbol], , , SEQ_DIST_SLOT); } while (symbol < DIST_SLOTS); -#else - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT0); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT1); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT2); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT3); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT4); - rc_bit_case(probs[symbol], , , SEQ_DIST_SLOT5); -#endif - // Get rid of the highest bit that was needed for - // indexing of the probability array. + symbol -= DIST_SLOTS; assert(symbol <= 63); if (symbol < DIST_MODEL_START) { - // Match distances [0, 3] have only two bits. rep0 = symbol; } else { - // Decode the lowest [1, 29] bits of - // the match distance. limit = (symbol >> 1) - 1; assert(limit >= 1 && limit <= 30); rep0 = 2 + (symbol & 1); if (symbol < DIST_MODEL_END) { - // Prepare to decode the low bits for - // a distance of [4, 127]. assert(limit <= 5); rep0 <<= limit; assert(rep0 <= 96); @@ -607,95 +809,36 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, symbol = 1; offset = 0; case SEQ_DIST_MODEL: -#ifdef HAVE_SMALL do { - rc_bit(probs[symbol], , + rc_bit_safe(probs[symbol], , rep0 += 1U << offset, SEQ_DIST_MODEL); } while (++offset < limit); -#else - switch (limit) { - case 5: - assert(offset == 0); - rc_bit(probs[symbol], , - rep0 += 1U, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 4: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 3: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 2: - rc_bit(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - ++offset; - --limit; - case 1: - // We need "symbol" only for - // indexing the probability - // array, thus we can use - // rc_bit_last() here to omit - // the unneeded updating of - // "symbol". - rc_bit_last(probs[symbol], , - rep0 += 1U << offset, - SEQ_DIST_MODEL); - } -#endif } else { - // The distance is >= 128. Decode the - // lower bits without probabilities - // except the lowest four bits. assert(symbol >= 14); assert(limit >= 6); limit -= ALIGN_BITS; assert(limit >= 2); case SEQ_DIRECT: - // Not worth manual unrolling - do { - rc_direct(rep0, SEQ_DIRECT); - } while (--limit > 0); + rc_direct_safe(rep0, limit, + SEQ_DIRECT); - // Decode the lowest four bits using - // probabilities. rep0 <<= ALIGN_BITS; - symbol = 1; -#ifdef HAVE_SMALL - offset = 0; + symbol = 0; + offset = 1; case SEQ_ALIGN: do { - rc_bit(coder->pos_align[ - symbol], , - rep0 += 1U << offset, + rc_bit_last_safe( + coder->pos_align[ + offset + + symbol], + , + symbol += offset, SEQ_ALIGN); - } while (++offset < ALIGN_BITS); -#else - case SEQ_ALIGN0: - rc_bit(coder->pos_align[symbol], , - rep0 += 1, SEQ_ALIGN0); - case SEQ_ALIGN1: - rc_bit(coder->pos_align[symbol], , - rep0 += 2, SEQ_ALIGN1); - case SEQ_ALIGN2: - rc_bit(coder->pos_align[symbol], , - rep0 += 4, SEQ_ALIGN2); - case SEQ_ALIGN3: - // Like in SEQ_DIST_MODEL, we don't - // need "symbol" for anything else - // than indexing the probability array. - rc_bit_last(coder->pos_align[symbol], , - rep0 += 8, SEQ_ALIGN3); -#endif + offset <<= 1; + } while (offset < ALIGN_SIZE); + + rep0 += symbol; if (rep0 == UINT32_MAX) { // End of payload marker was @@ -710,6 +853,9 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, // that EOPM might be used // (it's not allowed in // LZMA2). +#ifndef HAVE_SMALL +eopm: +#endif if (!eopm_is_valid) { ret = LZMA_DATA_ERROR; goto out; @@ -718,7 +864,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, case SEQ_EOPM: // LZMA1 stream with // end-of-payload marker. - rc_normalize(SEQ_EOPM); + rc_normalize_safe(SEQ_EOPM); ret = rc_is_finished(rc) ? LZMA_STREAM_END : LZMA_DATA_ERROR; @@ -727,36 +873,30 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, } } - // Validate the distance we just decoded. if (unlikely(!dict_is_distance_valid(&dict, rep0))) { ret = LZMA_DATA_ERROR; goto out; } } else { + ///////////////////// + // Repeated match. // + ///////////////////// + rc_update_1(coder->is_rep[state]); - // Repeated match - // - // The match distance is a value that we have had - // earlier. The latest four match distances are - // available as rep0, rep1, rep2 and rep3. We will - // now decode which of them is the new distance. - // - // There cannot be a match if we haven't produced - // any output, so check that first. if (unlikely(!dict_is_distance_valid(&dict, 0))) { ret = LZMA_DATA_ERROR; goto out; } case SEQ_IS_REP0: - rc_if_0(coder->is_rep0[state], SEQ_IS_REP0) { + rc_if_0_safe(coder->is_rep0[state], SEQ_IS_REP0) { rc_update_0(coder->is_rep0[state]); - // The distance is rep0. case SEQ_IS_REP0_LONG: - rc_if_0(coder->is_rep0_long[state][pos_state], + rc_if_0_safe(coder->is_rep0_long + [state][pos_state], SEQ_IS_REP0_LONG) { rc_update_0(coder->is_rep0_long[ state][pos_state]); @@ -764,8 +904,9 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, update_short_rep(state); case SEQ_SHORTREP: - if (unlikely(dict_put(&dict, dict_get( - &dict, rep0)))) { + if (dict_put_safe(&dict, + dict_get(&dict, + rep0))) { coder->sequence = SEQ_SHORTREP; goto out; } @@ -773,8 +914,6 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, continue; } - // Repeating more than one byte at - // distance of rep0. rc_update_1(coder->is_rep0_long[ state][pos_state]); @@ -782,11 +921,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, rc_update_1(coder->is_rep0[state]); case SEQ_IS_REP1: - // The distance is rep1, rep2 or rep3. Once - // we find out which one of these three, it - // is stored to rep0 and rep1, rep2 and rep3 - // are updated accordingly. - rc_if_0(coder->is_rep1[state], SEQ_IS_REP1) { + rc_if_0_safe(coder->is_rep1[state], SEQ_IS_REP1) { rc_update_0(coder->is_rep1[state]); const uint32_t distance = rep1; @@ -796,7 +931,7 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, } else { rc_update_1(coder->is_rep1[state]); case SEQ_IS_REP2: - rc_if_0(coder->is_rep2[state], + rc_if_0_safe(coder->is_rep2[state], SEQ_IS_REP2) { rc_update_0(coder->is_rep2[ state]); @@ -821,7 +956,6 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, update_long_rep(state); - // Decode the length of the repeated match. len_decode(len, coder->rep_len_decoder, pos_state, SEQ_REP_LEN); } @@ -830,13 +964,10 @@ lzma_decode(void *coder_ptr, lzma_dict *restrict dictptr, // Repeat from history buffer. // ///////////////////////////////// - // The length is always between these limits. There is no way - // to trigger the algorithm to set len outside this range. assert(len >= MATCH_LEN_MIN); assert(len <= MATCH_LEN_MAX); case SEQ_COPY: - // Repeat len bytes from distance of rep0. if (unlikely(dict_repeat(&dict, rep0, &len))) { coder->sequence = SEQ_COPY; goto out; @@ -890,7 +1021,6 @@ out: } - static void lzma_decoder_uncompressed(void *coder_ptr, lzma_vli uncompressed_size, bool allow_eopm) @@ -917,7 +1047,7 @@ lzma_decoder_reset(void *coder_ptr, const void *opt) literal_init(coder->literal, options->lc, options->lp); coder->literal_context_bits = options->lc; - coder->literal_pos_mask = (1U << options->lp) - 1; + coder->literal_mask = literal_mask_calc(options->lc, options->lp); // State coder->state = STATE_LIT_LIT; diff --git a/src/liblzma/lzma/lzma_decoder.h b/src/liblzma/lzma/lzma_decoder.h index 1427bc2..9730f56 100644 --- a/src/liblzma/lzma/lzma_decoder.h +++ b/src/liblzma/lzma/lzma_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_DECODER_H diff --git a/src/liblzma/lzma/lzma_encoder.c b/src/liblzma/lzma/lzma_encoder.c index 559c63e..543ca32 100644 --- a/src/liblzma/lzma/lzma_encoder.c +++ b/src/liblzma/lzma/lzma_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma2_encoder.h" @@ -49,24 +48,24 @@ literal(lzma_lzma1_encoder *coder, lzma_mf *mf, uint32_t position) const uint8_t cur_byte = mf->buffer[ mf->read_pos - mf->read_ahead]; probability *subcoder = literal_subcoder(coder->literal, - coder->literal_context_bits, coder->literal_pos_mask, + coder->literal_context_bits, coder->literal_mask, position, mf->buffer[mf->read_pos - mf->read_ahead - 1]); if (is_literal_state(coder->state)) { // Previous LZMA-symbol was a literal. Encode a normal // literal without a match byte. + update_literal_normal(coder->state); rc_bittree(&coder->rc, subcoder, 8, cur_byte); } else { // Previous LZMA-symbol was a match. Use the last byte of // the match as a "match byte". That is, compare the bits // of the current literal and the match byte. + update_literal_matched(coder->state); const uint8_t match_byte = mf->buffer[ mf->read_pos - coder->reps[0] - 1 - mf->read_ahead]; literal_matched(&coder->rc, subcoder, match_byte, cur_byte); } - - update_literal(coder->state); } @@ -283,7 +282,7 @@ encode_init(lzma_lzma1_encoder *coder, lzma_mf *mf) mf_skip(mf, 1); mf->read_ahead = 0; rc_bit(&coder->rc, &coder->is_match[0][0], 0); - rc_bittree(&coder->rc, coder->literal[0], 8, mf->buffer[0]); + rc_bittree(&coder->rc, coder->literal + 0, 8, mf->buffer[0]); ++coder->uncomp_size; } @@ -535,7 +534,7 @@ lzma_lzma_encoder_reset(lzma_lzma1_encoder *coder, coder->pos_mask = (1U << options->pb) - 1; coder->literal_context_bits = options->lc; - coder->literal_pos_mask = (1U << options->lp) - 1; + coder->literal_mask = literal_mask_calc(options->lc, options->lp); // Range coder rc_reset(&coder->rc); @@ -712,6 +711,9 @@ static lzma_ret lzma_encoder_init(lzma_lz_encoder *lz, const lzma_allocator *allocator, lzma_vli id, const void *options, lzma_lz_options *lz_options) { + if (options == NULL) + return LZMA_PROG_ERROR; + lz->code = &lzma_encode; lz->set_out_limit = &lzma_lzma_set_out_limit; return lzma_lzma_encoder_create( diff --git a/src/liblzma/lzma/lzma_encoder.h b/src/liblzma/lzma/lzma_encoder.h index 84d8c91..e8ae807 100644 --- a/src/liblzma/lzma/lzma_encoder.h +++ b/src/liblzma/lzma/lzma_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_ENCODER_H diff --git a/src/liblzma/lzma/lzma_encoder_optimum_fast.c b/src/liblzma/lzma/lzma_encoder_optimum_fast.c index 6c53d2b..0f063d5 100644 --- a/src/liblzma/lzma/lzma_encoder_optimum_fast.c +++ b/src/liblzma/lzma/lzma_encoder_optimum_fast.c @@ -1,12 +1,11 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_optimum_fast.c // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" diff --git a/src/liblzma/lzma/lzma_encoder_optimum_normal.c b/src/liblzma/lzma/lzma_encoder_optimum_normal.c index 101c8d4..a6c0398 100644 --- a/src/liblzma/lzma/lzma_encoder_optimum_normal.c +++ b/src/liblzma/lzma/lzma_encoder_optimum_normal.c @@ -1,12 +1,11 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_optimum_normal.c // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "lzma_encoder_private.h" @@ -24,7 +23,7 @@ get_literal_price(const lzma_lzma1_encoder *const coder, const uint32_t pos, uint32_t match_byte, uint32_t symbol) { const probability *const subcoder = literal_subcoder(coder->literal, - coder->literal_context_bits, coder->literal_pos_mask, + coder->literal_context_bits, coder->literal_mask, pos, prev_byte); uint32_t price = 0; diff --git a/src/liblzma/lzma/lzma_encoder_presets.c b/src/liblzma/lzma/lzma_encoder_presets.c index 711df02..e53483f 100644 --- a/src/liblzma/lzma/lzma_encoder_presets.c +++ b/src/liblzma/lzma/lzma_encoder_presets.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_presets.c @@ -6,9 +8,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "common.h" diff --git a/src/liblzma/lzma/lzma_encoder_private.h b/src/liblzma/lzma/lzma_encoder_private.h index b228c57..eeea5e9 100644 --- a/src/liblzma/lzma/lzma_encoder_private.h +++ b/src/liblzma/lzma/lzma_encoder_private.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzma_encoder_private.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_LZMA_ENCODER_PRIVATE_H @@ -116,10 +115,10 @@ struct lzma_lzma1_encoder_s { uint32_t pos_mask; ///< (1 << pos_bits) - 1 uint32_t literal_context_bits; - uint32_t literal_pos_mask; + uint32_t literal_mask; // These are the same as in lzma_decoder.c. See comments there. - probability literal[LITERAL_CODERS_MAX][LITERAL_CODER_SIZE]; + probability literal[LITERAL_CODERS_MAX * LITERAL_CODER_SIZE]; probability is_match[STATES][POS_STATES_MAX]; probability is_rep[STATES]; probability is_rep0[STATES]; diff --git a/src/liblzma/rangecoder/Makefile.inc b/src/liblzma/rangecoder/Makefile.inc index d8a597a..9a00e6e 100644 --- a/src/liblzma/rangecoder/Makefile.inc +++ b/src/liblzma/rangecoder/Makefile.inc @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## EXTRA_DIST += rangecoder/price_tablegen.c diff --git a/src/liblzma/rangecoder/price.h b/src/liblzma/rangecoder/price.h index 45dbbbb..cce6bda 100644 --- a/src/liblzma/rangecoder/price.h +++ b/src/liblzma/rangecoder/price.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file price.h @@ -5,9 +7,6 @@ // // Author: Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_PRICE_H diff --git a/src/liblzma/rangecoder/price_table.c b/src/liblzma/rangecoder/price_table.c index ac64bf6..c33433f 100644 --- a/src/liblzma/rangecoder/price_table.c +++ b/src/liblzma/rangecoder/price_table.c @@ -1,4 +1,6 @@ -/* This file has been automatically generated by price_tablegen.c. */ +// SPDX-License-Identifier: 0BSD + +// This file has been generated by price_tablegen.c. #include "range_encoder.h" diff --git a/src/liblzma/rangecoder/price_tablegen.c b/src/liblzma/rangecoder/price_tablegen.c index bf08ce3..4b6ca37 100644 --- a/src/liblzma/rangecoder/price_tablegen.c +++ b/src/liblzma/rangecoder/price_tablegen.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file price_tablegen.c @@ -8,13 +10,15 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include <inttypes.h> #include <stdio.h> + +// Make it compile without common.h. +#define BUILDING_PRICE_TABLEGEN +#define lzma_attr_visibility_hidden + #include "range_common.h" #include "price.h" @@ -54,11 +58,13 @@ init_price_table(void) static void print_price_table(void) { - printf("/* This file has been automatically generated by " - "price_tablegen.c. */\n\n" - "#include \"range_encoder.h\"\n\n" - "const uint8_t lzma_rc_prices[" - "RC_PRICE_TABLE_SIZE] = {"); + // Split the SPDX string so that it won't accidentally match + // when tools search for the string. + printf("// SPDX" "-License-Identifier" ": 0BSD\n\n" + "// This file has been generated by price_tablegen.c.\n\n" + "#include \"range_encoder.h\"\n\n" + "const uint8_t lzma_rc_prices[" + "RC_PRICE_TABLE_SIZE] = {"); const size_t array_size = sizeof(lzma_rc_prices) / sizeof(lzma_rc_prices[0]); diff --git a/src/liblzma/rangecoder/range_common.h b/src/liblzma/rangecoder/range_common.h index 2c74dc1..ac4dbe1 100644 --- a/src/liblzma/rangecoder/range_common.h +++ b/src/liblzma/rangecoder/range_common.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_common.h @@ -6,15 +8,15 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_COMMON_H #define LZMA_RANGE_COMMON_H -#include "common.h" +// Skip common.h if building price_tablegen.c. +#ifndef BUILDING_PRICE_TABLEGEN +# include "common.h" +#endif /////////////// @@ -66,6 +68,10 @@ /// /// I will be sticking to uint16_t unless some specific architectures /// are *much* faster (20-50 %) with uint32_t. +/// +/// Update in 2024: The branchless C and x86-64 assembly was written so that +/// probability is assumed to be uint16_t. (In contrast, LZMA SDK 23.01 +/// assembly supports both types.) typedef uint16_t probability; #endif diff --git a/src/liblzma/rangecoder/range_decoder.h b/src/liblzma/rangecoder/range_decoder.h index e0b051f..31a58d1 100644 --- a/src/liblzma/rangecoder/range_decoder.h +++ b/src/liblzma/rangecoder/range_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_decoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_DECODER_H @@ -17,6 +16,55 @@ #include "range_common.h" +// Choose the range decoder variants to use using a bitmask. +// If no bits are set, only the basic version is used. +// If more than one version is selected for the same feature, +// the last one on the list below is used. +// +// Bitwise-or of the following enable branchless C versions: +// 0x01 normal bittrees +// 0x02 fixed-sized reverse bittrees +// 0x04 variable-sized reverse bittrees (not faster) +// 0x08 matched literal (not faster) +// +// GCC & Clang compatible x86-64 inline assembly: +// 0x010 normal bittrees +// 0x020 fixed-sized reverse bittrees +// 0x040 variable-sized reverse bittrees +// 0x080 matched literal +// 0x100 direct bits +// +// The default can be overridden at build time by defining +// LZMA_RANGE_DECODER_CONFIG to the desired mask. +// +// 2024-02-22: Feedback from benchmarks: +// - Brancless C (0x003) can be better than basic on x86-64 but often it's +// slightly worse on other archs. Since asm is much better on x86-64, +// branchless C is not used at all. +// - With x86-64 asm, there are slight differences between GCC and Clang +// and different processors. Overall 0x1F0 seems to be the best choice. +#ifndef LZMA_RANGE_DECODER_CONFIG +# if defined(__x86_64__) && !defined(__ILP32__) \ + && !defined(__NVCOMPILER) \ + && (defined(__GNUC__) || defined(__clang__)) +# define LZMA_RANGE_DECODER_CONFIG 0x1F0 +# else +# define LZMA_RANGE_DECODER_CONFIG 0 +# endif +#endif + + +// Negative RC_BIT_MODEL_TOTAL but the lowest RC_MOVE_BITS are flipped. +// This is useful for updating probability variables in branchless decoding: +// +// uint32_t decoded_bit = ...; +// probability tmp = RC_BIT_MODEL_OFFSET; +// tmp &= decoded_bit - 1; +// prob -= (prob + tmp) >> RC_MOVE_BITS; +#define RC_BIT_MODEL_OFFSET \ + ((UINT32_C(1) << RC_MOVE_BITS) - 1 - RC_BIT_MODEL_TOTAL) + + typedef struct { uint32_t range; uint32_t code; @@ -50,18 +98,28 @@ rc_read_init(lzma_range_decoder *rc, const uint8_t *restrict in, /// Makes local copies of range decoder and *in_pos variables. Doing this /// improves speed significantly. The range decoder macros expect also -/// variables `in' and `in_size' to be defined. -#define rc_to_local(range_decoder, in_pos) \ +/// variables 'in' and 'in_size' to be defined. +#define rc_to_local(range_decoder, in_pos, fast_mode_in_required) \ lzma_range_decoder rc = range_decoder; \ - size_t rc_in_pos = (in_pos); \ + const uint8_t *rc_in_ptr = in + (in_pos); \ + const uint8_t *rc_in_end = in + in_size; \ + const uint8_t *rc_in_fast_end \ + = (rc_in_end - rc_in_ptr) <= (fast_mode_in_required) \ + ? rc_in_ptr \ + : rc_in_end - (fast_mode_in_required); \ + (void)rc_in_fast_end; /* Silence a warning with HAVE_SMALL. */ \ uint32_t rc_bound +/// Evaluates to true if there is enough input remaining to use fast mode. +#define rc_is_fast_allowed() (rc_in_ptr < rc_in_fast_end) + + /// Stores the local copes back to the range decoder structure. #define rc_from_local(range_decoder, in_pos) \ do { \ range_decoder = rc; \ - in_pos = rc_in_pos; \ + in_pos = (size_t)(rc_in_ptr - in); \ } while (0) @@ -81,18 +139,30 @@ do { \ ((range_decoder).code == 0) -/// Read the next input byte if needed. If more input is needed but there is +// Read the next input byte if needed. +#define rc_normalize() \ +do { \ + if (rc.range < RC_TOP_VALUE) { \ + rc.range <<= RC_SHIFT_BITS; \ + rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \ + } \ +} while (0) + + +/// If more input is needed but there is /// no more input available, "goto out" is used to jump out of the main -/// decoder loop. -#define rc_normalize(seq) \ +/// decoder loop. The "_safe" macros are used in the Resumable decoder +/// mode in order to save the sequence to continue decoding from that +/// point later. +#define rc_normalize_safe(seq) \ do { \ if (rc.range < RC_TOP_VALUE) { \ - if (unlikely(rc_in_pos == in_size)) { \ + if (rc_in_ptr == rc_in_end) { \ coder->sequence = seq; \ goto out; \ } \ rc.range <<= RC_SHIFT_BITS; \ - rc.code = (rc.code << RC_SHIFT_BITS) | in[rc_in_pos++]; \ + rc.code = (rc.code << RC_SHIFT_BITS) | *rc_in_ptr++; \ } \ } while (0) @@ -100,7 +170,7 @@ do { \ /// Start decoding a bit. This must be used together with rc_update_0() /// and rc_update_1(): /// -/// rc_if_0(prob, seq) { +/// rc_if_0(prob) { /// rc_update_0(prob); /// // Do something /// } else { @@ -108,18 +178,28 @@ do { \ /// // Do something else /// } /// -#define rc_if_0(prob, seq) \ - rc_normalize(seq); \ +#define rc_if_0(prob) \ + rc_normalize(); \ + rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ + if (rc.code < rc_bound) + + +#define rc_if_0_safe(prob, seq) \ + rc_normalize_safe(seq); \ rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); \ if (rc.code < rc_bound) /// Update the range decoder state and the used probability variable to /// match a decoded bit of 0. +/// +/// The x86-64 assembly uses the commented method but it seems that, +/// at least on x86-64, the first version is slightly faster as C code. #define rc_update_0(prob) \ do { \ rc.range = rc_bound; \ prob += (RC_BIT_MODEL_TOTAL - (prob)) >> RC_MOVE_BITS; \ + /* prob -= ((prob) + RC_BIT_MODEL_OFFSET) >> RC_MOVE_BITS; */ \ } while (0) @@ -137,9 +217,21 @@ do { \ /// This macro is used as the last step in bittree reverse decoders since /// those don't use "symbol" for anything else than indexing the probability /// arrays. -#define rc_bit_last(prob, action0, action1, seq) \ +#define rc_bit_last(prob, action0, action1) \ +do { \ + rc_if_0(prob) { \ + rc_update_0(prob); \ + action0; \ + } else { \ + rc_update_1(prob); \ + action1; \ + } \ +} while (0) + + +#define rc_bit_last_safe(prob, action0, action1, seq) \ do { \ - rc_if_0(prob, seq) { \ + rc_if_0_safe(prob, seq) { \ rc_update_0(prob); \ action0; \ } else { \ @@ -151,35 +243,724 @@ do { \ /// Decodes one bit, updates "symbol", and runs action0 or action1 depending /// on the decoded bit. -#define rc_bit(prob, action0, action1, seq) \ +#define rc_bit(prob, action0, action1) \ rc_bit_last(prob, \ symbol <<= 1; action0, \ + symbol = (symbol << 1) + 1; action1); + + +#define rc_bit_safe(prob, action0, action1, seq) \ + rc_bit_last_safe(prob, \ + symbol <<= 1; action0, \ symbol = (symbol << 1) + 1; action1, \ seq); +// Unroll fixed-sized bittree decoding. +// +// A compile-time constant in final_add can be used to get rid of the high bit +// from symbol that is used for the array indexing (1U << bittree_bits). +// final_add may also be used to add offset to the result (LZMA length +// decoder does that). +// +// The reason to have final_add here is that in the asm code the addition +// can be done for free: in x86-64 there is SBB instruction with -1 as +// the immediate value, and final_add is combined with that value. +#define rc_bittree_bit(prob) \ + rc_bit(prob, , ) + +#define rc_bittree3(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) -/// Like rc_bit() but add "case seq:" as a prefix. This makes the unrolled -/// loops more readable because the code isn't littered with "case" -/// statements. On the other hand this also makes it less readable, since -/// spotting the places where the decoder loop may be restarted is less -/// obvious. -#define rc_bit_case(prob, action0, action1, seq) \ - case seq: rc_bit(prob, action0, action1, seq) +#define rc_bittree6(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) + +#define rc_bittree8(probs, final_add) \ +do { \ + symbol = 1; \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + rc_bittree_bit(probs[symbol]); \ + symbol += (uint32_t)(final_add); \ +} while (0) + + +// Fixed-sized reverse bittree +#define rc_bittree_rev4(probs) \ +do { \ + symbol = 0; \ + rc_bit_last(probs[symbol + 1], , symbol += 1); \ + rc_bit_last(probs[symbol + 2], , symbol += 2); \ + rc_bit_last(probs[symbol + 4], , symbol += 4); \ + rc_bit_last(probs[symbol + 8], , symbol += 8); \ +} while (0) + + +// Decode one bit from variable-sized reverse bittree. The loop is done +// in the code that uses this macro. This could be changed if the assembly +// version benefited from having the loop done in assembly but it didn't +// seem so in early 2024. +// +// Also, if the loop was done here, the loop counter would likely be local +// to the macro so that it wouldn't modify yet another input variable. +// If a _safe version of a macro with a loop was done then a modifiable +// input variable couldn't be avoided though. +#define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \ + rc_bit(probs[symbol], \ + , \ + dest += value_to_add_if_1); + + +// Matched literal +#define decode_with_match_bit \ + t_match_byte <<= 1; \ + t_match_bit = t_match_byte & t_offset; \ + t_subcoder_index = t_offset + t_match_bit + symbol; \ + rc_bit(probs[t_subcoder_index], \ + t_offset &= ~t_match_bit, \ + t_offset &= t_match_bit) + +#define rc_matched_literal(probs_base_var, match_byte) \ +do { \ + uint32_t t_match_byte = (match_byte); \ + uint32_t t_match_bit; \ + uint32_t t_subcoder_index; \ + uint32_t t_offset = 0x100; \ + symbol = 1; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ + decode_with_match_bit; \ +} while (0) /// Decode a bit without using a probability. -#define rc_direct(dest, seq) \ +// +// NOTE: GCC 13 and Clang/LLVM 16 can, at least on x86-64, optimize the bound +// calculation to use an arithmetic right shift so there's no need to provide +// the alternative code which, according to C99/C11/C23 6.3.1.3-p3 isn't +// perfectly portable: rc_bound = (uint32_t)((int32_t)rc.code >> 31); +#define rc_direct(dest, count_var) \ do { \ - rc_normalize(seq); \ + dest = (dest << 1) + 1; \ + rc_normalize(); \ + rc.range >>= 1; \ + rc.code -= rc.range; \ + rc_bound = UINT32_C(0) - (rc.code >> 31); \ + dest += rc_bound; \ + rc.code += rc.range & rc_bound; \ +} while (--count_var > 0) + + + +#define rc_direct_safe(dest, count_var, seq) \ +do { \ + rc_normalize_safe(seq); \ rc.range >>= 1; \ rc.code -= rc.range; \ rc_bound = UINT32_C(0) - (rc.code >> 31); \ rc.code += rc.range & rc_bound; \ dest = (dest << 1) + (rc_bound + 1); \ +} while (--count_var > 0) + + +////////////////// +// Branchless C // +////////////////// + +/// Decode a bit using a branchless method. This reduces the number of +/// mispredicted branches and thus can improve speed. +#define rc_c_bit(prob, action_bit, action_neg) \ +do { \ + probability *p = &(prob); \ + rc_normalize(); \ + rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * *p; \ + uint32_t rc_mask = rc.code >= rc_bound; /* rc_mask = decoded bit */ \ + action_bit; /* action when rc_mask is 0 or 1 */ \ + /* rc_mask becomes 0 if bit is 0 and 0xFFFFFFFF if bit is 1: */ \ + rc_mask = 0U - rc_mask; \ + rc.range &= rc_mask; /* If bit 0: set rc.range = 0 */ \ + rc_bound ^= rc_mask; \ + rc_bound -= rc_mask; /* If bit 1: rc_bound = 0U - rc_bound */ \ + rc.range += rc_bound; \ + rc_bound &= rc_mask; \ + rc.code += rc_bound; \ + action_neg; /* action when rc_mask is 0 or 0xFFFFFFFF */ \ + rc_mask = ~rc_mask; /* If bit 0: all bits are set in rc_mask */ \ + rc_mask &= RC_BIT_MODEL_OFFSET; \ + *p -= (*p + rc_mask) >> RC_MOVE_BITS; \ } while (0) -// NOTE: No macros are provided for bittree decoding. It seems to be simpler -// to just write them open in the code. +// Testing on x86-64 give an impression that only the normal bittrees and +// the fixed-sized reverse bittrees are worth the branchless C code. +// It should be tested on other archs for which there isn't assembly code +// in this file. + +// Using addition in "(symbol << 1) + rc_mask" allows use of x86 LEA +// or RISC-V SH1ADD instructions. Compilers might infer it from +// "(symbol << 1) | rc_mask" too if they see that mask is 0 or 1 but +// the use of addition doesn't require such analysis from compilers. +#if LZMA_RANGE_DECODER_CONFIG & 0x01 +#undef rc_bittree_bit +#define rc_bittree_bit(prob) \ + rc_c_bit(prob, \ + symbol = (symbol << 1) + rc_mask, \ + ) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x01 + +#if LZMA_RANGE_DECODER_CONFIG & 0x02 +#undef rc_bittree_rev4 +#define rc_bittree_rev4(probs) \ +do { \ + symbol = 0; \ + rc_c_bit(probs[symbol + 1], symbol += rc_mask, ); \ + rc_c_bit(probs[symbol + 2], symbol += rc_mask << 1, ); \ + rc_c_bit(probs[symbol + 4], symbol += rc_mask << 2, ); \ + rc_c_bit(probs[symbol + 8], symbol += rc_mask << 3, ); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x02 + +#if LZMA_RANGE_DECODER_CONFIG & 0x04 +#undef rc_bit_add_if_1 +#define rc_bit_add_if_1(probs, dest, value_to_add_if_1) \ + rc_c_bit(probs[symbol], \ + symbol = (symbol << 1) + rc_mask, \ + dest += (value_to_add_if_1) & rc_mask) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x04 + + +#if LZMA_RANGE_DECODER_CONFIG & 0x08 +#undef decode_with_match_bit +#define decode_with_match_bit \ + t_match_byte <<= 1; \ + t_match_bit = t_match_byte & t_offset; \ + t_subcoder_index = t_offset + t_match_bit + symbol; \ + rc_c_bit(probs[t_subcoder_index], \ + symbol = (symbol << 1) + rc_mask, \ + t_offset &= ~t_match_bit ^ rc_mask) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x08 + + +//////////// +// x86-64 // +//////////// + +#if LZMA_RANGE_DECODER_CONFIG & 0x1F0 + +// rc_asm_y and rc_asm_n are used as arguments to macros to control which +// strings to include or omit. +#define rc_asm_y(str) str +#define rc_asm_n(str) + +// There are a few possible variations for normalization. +// This is the smallest variant which is also used by LZMA SDK. +// +// - This has partial register write (the MOV from (%[in_ptr])). +// +// - INC saves one byte in code size over ADD. False dependency on +// partial flags from INC shouldn't become a problem on any processor +// because the instructions after normalization don't read the flags +// until SUB which sets all flags. +// +#define rc_asm_normalize \ + "cmp %[top_value], %[range]\n\t" \ + "jae 1f\n\t" \ + "shl %[shift_bits], %[code]\n\t" \ + "mov (%[in_ptr]), %b[code]\n\t" \ + "shl %[shift_bits], %[range]\n\t" \ + "inc %[in_ptr]\n" \ + "1:\n" + +// rc_asm_calc(prob) is roughly equivalent to the C version of rc_if_0(prob)... +// +// rc_bound = (rc.range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); +// if (rc.code < rc_bound) +// +// ...but the bound is stored in "range": +// +// t0 = range; +// range = (range >> RC_BIT_MODEL_TOTAL_BITS) * (prob); +// t0 -= range; +// t1 = code; +// code -= range; +// +// The carry flag (CF) from the last subtraction holds the negation of +// the decoded bit (if CF==0 then the decoded bit is 1). +// The values in t0 and t1 are needed for rc_update_0(prob) and +// rc_update_1(prob). If the bit is 0, rc_update_0(prob)... +// +// rc.range = rc_bound; +// +// ...has already been done but the "code -= range" has to be reverted using +// the old value stored in t1. (Also, prob needs to be updated.) +// +// If the bit is 1, rc_update_1(prob)... +// +// rc.range -= rc_bound; +// rc.code -= rc_bound; +// +// ...is already done for "code" but the value for "range" needs to be taken +// from t0. (Also, prob needs to be updated here as well.) +// +// The assignments from t0 and t1 can be done in a branchless manner with CMOV +// after the instructions from this macro. The CF from SUB tells which moves +// are needed. +#define rc_asm_calc(prob) \ + "mov %[range], %[t0]\n\t" \ + "shr %[bit_model_total_bits], %[range]\n\t" \ + "imul %[" prob "], %[range]\n\t" \ + "sub %[range], %[t0]\n\t" \ + "mov %[code], %[t1]\n\t" \ + "sub %[range], %[code]\n\t" + +// Also, prob needs to be updated: The update math depends on the decoded bit. +// It can be expressed in a few slightly different ways but this is fairly +// convenient here: +// +// prob -= (prob + (bit ? 0 : RC_BIT_MODEL_OFFSET)) >> RC_MOVE_BITS; +// +// To do it in branchless way when the negation of the decoded bit is in CF, +// both "prob" and "prob + RC_BIT_MODEL_OFFSET" are needed. Then the desired +// value can be picked with CMOV. The addition can be done using LEA without +// affecting CF. +// +// (This prob update method is a tiny bit different from LZMA SDK 23.01. +// In the LZMA SDK a single register is reserved solely for a constant to +// be used with CMOV when updating prob. That is fine since there are enough +// free registers to do so. The method used here uses one fewer register, +// which is valuable with inline assembly.) +// +// * * * +// +// In bittree decoding, each (unrolled) loop iteration decodes one bit +// and needs one prob variable. To make it faster, the prob variable of +// the iteration N+1 is loaded during iteration N. There are two possible +// prob variables to choose from for N+1. Both are loaded from memory and +// the correct one is chosen with CMOV using the same CF as is used for +// other things described above. +// +// This preloading/prefetching requires an extra register. To avoid +// useless moves from "preloaded prob register" to "current prob register", +// the macros swap between the two registers for odd and even iterations. +// +// * * * +// +// Finally, the decoded bit has to be stored in "symbol". Since the negation +// of the bit is in CF, this can be done with SBB: symbol -= CF - 1. That is, +// if the decoded bit is 0 (CF==1) the operation is a no-op "symbol -= 0" +// and when bit is 1 (CF==0) the operation is "symbol -= 0 - 1" which is +// the same as "symbol += 1". +// +// The instructions for all things are intertwined for a few reasons: +// - freeing temporary registers for new use +// - not modifying CF too early +// - instruction scheduling +// +// The first and last iterations can cheat a little. For example, +// on the first iteration "symbol" is known to start from 1 so it +// doesn't need to be read; it can even be immediately initialized +// to 2 to prepare for the second iteration of the loop. +// +// * * * +// +// a = number of the current prob variable (0 or 1) +// b = number of the next prob variable (1 or 0) +// *_only = rc_asm_y or _n to include or exclude code marked with them +#define rc_asm_bittree(a, b, first_only, middle_only, last_only) \ + first_only( \ + "movzw 2(%[probs_base]), %[prob" #a "]\n\t" \ + "mov $2, %[symbol]\n\t" \ + "movzw 4(%[probs_base]), %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + /* Note the scaling of 4 instead of 2: */ \ + "movzw (%[probs_base], %q[symbol], 4), %[prob" #b "]\n\t" \ + ) \ + last_only( \ + "add %[symbol], %[symbol]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob" #a) \ + \ + "cmovae %[t0], %[range]\n\t" \ + \ + first_only( \ + "movzw 6(%[probs_base]), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzw 2(%[probs_base], %q[symbol], 4), %[t0]\n\t" \ + "lea (%q[symbol], %q[symbol]), %[symbol]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + \ + "lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "mov %[symbol], %[t1]\n\t" \ + "cmovae %[prob" #a "], %[t0]\n\t" \ + \ + first_only( \ + "sbb $-1, %[symbol]\n\t" \ + ) \ + middle_only( \ + "sbb $-1, %[symbol]\n\t" \ + ) \ + last_only( \ + "sbb %[last_sbb], %[symbol]\n\t" \ + ) \ + \ + "shr %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob" #a "]\n\t" \ + /* Scaling of 1 instead of 2 because symbol <<= 1. */ \ + "mov %w[prob" #a "], (%[probs_base], %q[t1], 1)\n\t" + +// NOTE: The order of variables in __asm__ can affect speed and code size. +#define rc_asm_bittree_n(probs_base_var, final_add, asm_str) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t_prob0; \ + uint32_t t_prob1; \ + \ + __asm__( \ + asm_str \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob0] "=&r"(t_prob0), \ + [prob1] "=&r"(t_prob1), \ + [symbol] "=&r"(symbol), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [last_sbb] "n"(-1 - (final_add)), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) + + +#if LZMA_RANGE_DECODER_CONFIG & 0x010 +#undef rc_bittree3 +#define rc_bittree3(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) + +#undef rc_bittree6 +#define rc_bittree6(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) + +#undef rc_bittree8 +#define rc_bittree8(probs_base_var, final_add) \ + rc_asm_bittree_n(probs_base_var, final_add, \ + rc_asm_bittree(0, 1, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(0, 1, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree(1, 0, rc_asm_n, rc_asm_n, rc_asm_y) \ + ) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x010 + + +// Fixed-sized reverse bittree +// +// This uses the indexing that constructs the final value in symbol directly. +// add = 1, 2, 4, 8 +// dcur = -, 4, 8, 16 +// dnext0 = 4, 8, 16, - +// dnext0 = 6, 12, 24, - +#define rc_asm_bittree_rev(a, b, add, dcur, dnext0, dnext1, \ + first_only, middle_only, last_only) \ + first_only( \ + "movzw 2(%[probs_base]), %[prob" #a "]\n\t" \ + "xor %[symbol], %[symbol]\n\t" \ + "movzw 4(%[probs_base]), %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzw " #dnext0 "(%[probs_base], %q[symbol], 2), " \ + "%[prob" #b "]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob" #a) \ + \ + "cmovae %[t0], %[range]\n\t" \ + \ + first_only( \ + "movzw 6(%[probs_base]), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + middle_only( \ + "movzw " #dnext1 "(%[probs_base], %q[symbol], 2), %[t0]\n\t" \ + "cmovae %[t0], %[prob" #b "]\n\t" \ + ) \ + \ + "lea " #add "(%q[symbol]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + middle_only( \ + "mov %[symbol], %[t1]\n\t" \ + ) \ + last_only( \ + "mov %[symbol], %[t1]\n\t" \ + ) \ + "cmovae %[t0], %[symbol]\n\t" \ + "lea %c[bit_model_offset](%q[prob" #a "]), %[t0]\n\t" \ + "cmovae %[prob" #a "], %[t0]\n\t" \ + \ + "shr %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob" #a "]\n\t" \ + first_only( \ + "mov %w[prob" #a "], 2(%[probs_base])\n\t" \ + ) \ + middle_only( \ + "mov %w[prob" #a "], " \ + #dcur "(%[probs_base], %q[t1], 2)\n\t" \ + ) \ + last_only( \ + "mov %w[prob" #a "], " \ + #dcur "(%[probs_base], %q[t1], 2)\n\t" \ + ) + +#if LZMA_RANGE_DECODER_CONFIG & 0x020 +#undef rc_bittree_rev4 +#define rc_bittree_rev4(probs_base_var) \ +rc_asm_bittree_n(probs_base_var, 4, \ + rc_asm_bittree_rev(0, 1, 1, -, 4, 6, rc_asm_y, rc_asm_n, rc_asm_n) \ + rc_asm_bittree_rev(1, 0, 2, 4, 8, 12, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree_rev(0, 1, 4, 8, 16, 24, rc_asm_n, rc_asm_y, rc_asm_n) \ + rc_asm_bittree_rev(1, 0, 8, 16, -, -, rc_asm_n, rc_asm_n, rc_asm_y) \ +) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x020 + + +#if LZMA_RANGE_DECODER_CONFIG & 0x040 +#undef rc_bit_add_if_1 +#define rc_bit_add_if_1(probs_base_var, dest_var, value_to_add_if_1) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t2 = (value_to_add_if_1); \ + uint32_t t_prob; \ + uint32_t t_index; \ + \ + __asm__( \ + "movzw (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ + "mov %[symbol], %[index]\n\t" \ + \ + "add %[dest], %[t2]\n\t" \ + "add %[symbol], %[symbol]\n\t" \ + \ + rc_asm_normalize \ + rc_asm_calc("prob") \ + \ + "cmovae %[t0], %[range]\n\t" \ + "lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "cmovae %[prob], %[t0]\n\t" \ + \ + "cmovae %[t2], %[dest]\n\t" \ + "sbb $-1, %[symbol]\n\t" \ + \ + "sar %[move_bits], %[t0]\n\t" \ + "sub %[t0], %[prob]\n\t" \ + "mov %w[prob], (%[probs_base], %q[index], 2)" \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob] "=&r"(t_prob), \ + [index] "=&r"(t_index), \ + [symbol] "+&r"(symbol), \ + [t2] "+&r"(t2), \ + [dest] "+&r"(dest_var), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x040 + + +// Literal decoding uses a normal 8-bit bittree but literal with match byte +// is more complex in picking the probability variable from the correct +// subtree. This doesn't use preloading/prefetching of the next prob because +// there are four choices instead of two. +// +// FIXME? The first iteration starts with symbol = 1 so it could be optimized +// by a tiny amount. +#define rc_asm_matched_literal(nonlast_only) \ + "add %[offset], %[symbol]\n\t" \ + "and %[offset], %[match_bit]\n\t" \ + "add %[match_bit], %[symbol]\n\t" \ + \ + "movzw (%[probs_base], %q[symbol], 2), %[prob]\n\t" \ + \ + "add %[symbol], %[symbol]\n\t" \ + \ + nonlast_only( \ + "xor %[match_bit], %[offset]\n\t" \ + "add %[match_byte], %[match_byte]\n\t" \ + ) \ + \ + rc_asm_normalize \ + rc_asm_calc("prob") \ + \ + "cmovae %[t0], %[range]\n\t" \ + "lea %c[bit_model_offset](%q[prob]), %[t0]\n\t" \ + "cmovb %[t1], %[code]\n\t" \ + "mov %[symbol], %[t1]\n\t" \ + "cmovae %[prob], %[t0]\n\t" \ + \ + nonlast_only( \ + "cmovae %[match_bit], %[offset]\n\t" \ + "mov %[match_byte], %[match_bit]\n\t" \ + ) \ + \ + "sbb $-1, %[symbol]\n\t" \ + \ + "shr %[move_bits], %[t0]\n\t" \ + /* Undo symbol += match_bit + offset: */ \ + "and $0x1FF, %[symbol]\n\t" \ + "sub %[t0], %[prob]\n\t" \ + \ + /* Scaling of 1 instead of 2 because symbol <<= 1. */ \ + "mov %w[prob], (%[probs_base], %q[t1], 1)\n\t" + + +#if LZMA_RANGE_DECODER_CONFIG & 0x080 +#undef rc_matched_literal +#define rc_matched_literal(probs_base_var, match_byte_value) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + uint32_t t_prob; \ + uint32_t t_match_byte = (uint32_t)(match_byte_value) << 1; \ + uint32_t t_match_bit = t_match_byte; \ + uint32_t t_offset = 0x100; \ + symbol = 1; \ + \ + __asm__( \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_y) \ + rc_asm_matched_literal(rc_asm_n) \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [prob] "=&r"(t_prob), \ + [match_bit] "+&r"(t_match_bit), \ + [symbol] "+&r"(symbol), \ + [match_byte] "+&r"(t_match_byte), \ + [offset] "+&r"(t_offset), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [probs_base] "r"(probs_base_var), \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS), \ + [bit_model_total_bits] "n"(RC_BIT_MODEL_TOTAL_BITS), \ + [bit_model_offset] "n"(RC_BIT_MODEL_OFFSET), \ + [move_bits] "n"(RC_MOVE_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x080 + + +// Doing the loop in asm instead of C seems to help a little. +#if LZMA_RANGE_DECODER_CONFIG & 0x100 +#undef rc_direct +#define rc_direct(dest_var, count_var) \ +do { \ + uint32_t t0; \ + uint32_t t1; \ + \ + __asm__( \ + "2:\n\t" \ + "add %[dest], %[dest]\n\t" \ + "lea 1(%q[dest]), %[t1]\n\t" \ + \ + rc_asm_normalize \ + \ + "shr $1, %[range]\n\t" \ + "mov %[code], %[t0]\n\t" \ + "sub %[range], %[code]\n\t" \ + "cmovns %[t1], %[dest]\n\t" \ + "cmovs %[t0], %[code]\n\t" \ + "dec %[count]\n\t" \ + "jnz 2b\n\t" \ + : \ + [range] "+&r"(rc.range), \ + [code] "+&r"(rc.code), \ + [t0] "=&r"(t0), \ + [t1] "=&r"(t1), \ + [dest] "+&r"(dest_var), \ + [count] "+&r"(count_var), \ + [in_ptr] "+&r"(rc_in_ptr) \ + : \ + [top_value] "n"(RC_TOP_VALUE), \ + [shift_bits] "n"(RC_SHIFT_BITS) \ + : \ + "cc", "memory"); \ +} while (0) +#endif // LZMA_RANGE_DECODER_CONFIG & 0x100 + +#endif // x86_64 #endif diff --git a/src/liblzma/rangecoder/range_encoder.h b/src/liblzma/rangecoder/range_encoder.h index d794eab..8f62a47 100644 --- a/src/liblzma/rangecoder/range_encoder.h +++ b/src/liblzma/rangecoder/range_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file range_encoder.h @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_RANGE_ENCODER_H diff --git a/src/liblzma/simple/Makefile.inc b/src/liblzma/simple/Makefile.inc index dc092f9..f42ad1b 100644 --- a/src/liblzma/simple/Makefile.inc +++ b/src/liblzma/simple/Makefile.inc @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## liblzma_la_SOURCES += \ simple/simple_coder.c \ @@ -49,3 +45,7 @@ endif if COND_FILTER_SPARC liblzma_la_SOURCES += simple/sparc.c endif + +if COND_FILTER_RISCV +liblzma_la_SOURCES += simple/riscv.c +endif diff --git a/src/liblzma/simple/arm.c b/src/liblzma/simple/arm.c index 6e53970..58acb2d 100644 --- a/src/liblzma/simple/arm.c +++ b/src/liblzma/simple/arm.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file arm.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/arm64.c b/src/liblzma/simple/arm64.c index 0fe0824..0a73f6c 100644 --- a/src/liblzma/simple/arm64.c +++ b/src/liblzma/simple/arm64.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file arm64.c @@ -16,9 +18,6 @@ // Jia Tan // Igor Pavlov // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/armthumb.c b/src/liblzma/simple/armthumb.c index 25d8dbd..f1eeca9 100644 --- a/src/liblzma/simple/armthumb.c +++ b/src/liblzma/simple/armthumb.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file armthumb.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/ia64.c b/src/liblzma/simple/ia64.c index 692b0a2..5025014 100644 --- a/src/liblzma/simple/ia64.c +++ b/src/liblzma/simple/ia64.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file ia64.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/powerpc.c b/src/liblzma/simple/powerpc.c index 3a340fd..ba6cfbe 100644 --- a/src/liblzma/simple/powerpc.c +++ b/src/liblzma/simple/powerpc.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file powerpc.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/riscv.c b/src/liblzma/simple/riscv.c new file mode 100644 index 0000000..b18df8b --- /dev/null +++ b/src/liblzma/simple/riscv.c @@ -0,0 +1,755 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file riscv.c +/// \brief Filter for 32-bit/64-bit little/big endian RISC-V binaries +/// +/// This converts program counter relative addresses in function calls +/// (JAL, AUIPC+JALR), address calculation of functions and global +/// variables (AUIPC+ADDI), loads (AUIPC+load), and stores (AUIPC+store). +/// +/// For AUIPC+inst2 pairs, the paired instruction checking is fairly relaxed. +/// The paired instruction opcode must only have its lowest two bits set, +/// meaning it will convert any paired instruction that is not a 16-bit +/// compressed instruction. This was shown to be enough to keep the number +/// of false matches low while improving code size and speed. +// +// Authors: Lasse Collin +// Jia Tan +// +// Special thanks: +// +// - Chien Wong <m@xv97.com> provided a few early versions of RISC-V +// filter variants along with test files and benchmark results. +// +// - Igor Pavlov helped a lot in the filter design, getting it both +// faster and smaller. The implementation here is still independently +// written, not based on LZMA SDK. +// +/////////////////////////////////////////////////////////////////////////////// + +/* + +RISC-V filtering +================ + + RV32I and RV64I, possibly combined with extensions C, Zfh, F, D, + and Q, are identical enough that the same filter works for both. + + The instruction encoding is always little endian, even on systems + with big endian data access. Thus the same filter works for both + endiannesses. + + The following instructions have program counter relative + (pc-relative) behavior: + +JAL +--- + + JAL is used for function calls (including tail calls) and + unconditional jumps within functions. Jumps within functions + aren't useful to filter because the absolute addresses often + appear only once or at most a few times. Tail calls and jumps + within functions look the same to a simple filter so neither + are filtered, that is, JAL x0 is ignored (the ABI name of the + register x0 is "zero"). + + Almost all calls store the return address to register x1 (ra) + or x5 (t0). To reduce false matches when the filter is applied + to non-code data, only the JAL instructions that use x1 or x5 + are converted. JAL has pc-relative range of +/-1 MiB so longer + calls and jumps need another method (AUIPC+JALR). + +C.J and C.JAL +------------- + + C.J and C.JAL have pc-relative range of +/-2 KiB. + + C.J is for tail calls and jumps within functions and isn't + filtered for the reasons mentioned for JAL x0. + + C.JAL is an RV32C-only instruction. Its encoding overlaps with + RV64C-only C.ADDIW which is a common instruction. So if filtering + C.JAL was useful (it wasn't tested) then a separate filter would + be needed for RV32 and RV64. Also, false positives would be a + significant problem when the filter is applied to non-code data + because C.JAL needs only five bits to match. Thus, this filter + doesn't modify C.JAL instructions. + +BEQ, BNE, BLT, BGE, BLTU, BGEU, C.BEQZ, and C.BNEZ +-------------------------------------------------- + + These are conditional branches with pc-relative range + of +/-4 KiB (+/-256 B for C.*). The absolute addresses often + appear only once and very short distances are the most common, + so filtering these instructions would make compression worse. + +AUIPC with rd != x0 +------------------- + + AUIPC is paired with a second instruction (inst2) to do + pc-relative jumps, calls, loads, stores, and for taking + an address of a symbol. AUIPC has a 20-bit immediate and + the possible inst2 choices have a 12-bit immediate. + + AUIPC stores pc + 20-bit signed immediate to a register. + The immediate encodes a multiple of 4 KiB so AUIPC itself + has a pc-relative range of +/-2 GiB. AUIPC does *NOT* set + the lowest 12 bits of the result to zero! This means that + the 12-bit immediate in inst2 cannot just include the lowest + 12 bits of the absolute address as is; the immediate has to + compensate for the lowest 12 bits that AUIPC copies from the + program counter. This means that a good filter has to convert + not only AUIPC but also the paired inst2. + + A strict filter would focus on filtering the following + AUIPC+inst2 pairs: + + - AUIPC+JALR: Function calls, including tail calls. + + - AUIPC+ADDI: Calculating the address of a function + or a global variable. + + - AUIPC+load/store from the base instruction sets + (RV32I, RV64I) or from the floating point extensions + Zfh, F, D, and Q: + * RV32I: LB, LH, LW, LBU, LHU, SB, SH, SW + * RV64I has also: LD, LWU, SD + * Zfh: FLH, FSH + * F: FLW, FSW + * D: FLD, FSD + * Q: FLQ, FSQ + + NOTE: AUIPC+inst2 can only be a pair if AUIPC's rd specifies + the same register as inst2's rs1. + + Instead of strictly accepting only the above instructions as inst2, + this filter uses a much simpler condition: the lowest two bits of + inst2 must be set, that is, inst2 must not be a 16-bit compressed + instruction. So this will accept all 32-bit and possible future + extended instructions as a pair to AUIPC if the bits in AUIPC's + rd [11:7] match the bits [19:15] in inst2 (the bits that I-type and + S-type instructions use for rs1). Testing showed that this relaxed + condition for inst2 did not consistently or significantly affect + compression ratio but it reduced code size and improved speed. + + Additionally, the paired instruction is always treated as an I-type + instruction. The S-type instructions used by stores (SB, SH, SW, + etc.) place the lowest 5 bits of the immediate in a different + location than I-type instructions. AUIPC+store pairs are less + common than other pairs, and testing showed that the extra + code required to handle S-type instructions was not worth the + compression ratio gained. + + AUIPC+inst2 don't necessarily appear sequentially next to each + other although very often they do. Especially AUIPC+JALR are + sequential as that may allow instruction fusion in processors + (and perhaps help branch prediction as a fused AUIPC+JALR is + a direct branch while JALR alone is an indirect branch). + + Clang 16 can generate code where AUIPC+inst2 is split: + + - AUIPC is outside a loop and inst2 (load/store) is inside + the loop. This way the AUIPC instruction needs to be + executed only once. + + - Load-modify-store may have AUIPC for the load and the same + AUIPC-result is used for the store too. This may get combined + with AUIPC being outside the loop. + + - AUIPC is before a conditional branch and inst2 is hundreds + of bytes away at the branch target. + + - Inner and outer pair: + + auipc a1,0x2f + auipc a2,0x3d + ld a2,-500(a2) + addi a1,a1,-233 + + - Many split pairs with an untaken conditional branch between: + + auipc s9,0x1613 # Pair 1 + auipc s4,0x1613 # Pair 2 + auipc s6,0x1613 # Pair 3 + auipc s10,0x1613 # Pair 4 + beqz a5,a3baae + ld a0,0(a6) + ld a6,246(s9) # Pair 1 + ld a1,250(s4) # Pair 2 + ld a3,254(s6) # Pair 3 + ld a4,258(s10) # Pair 4 + + It's not possible to find all split pairs in a filter like this. + At least in 2024, simple sequential pairs are 99 % of AUIPC uses + so filtering only such pairs gives good results and makes the + filter simpler. However, it's possible that future compilers will + produce different code where sequential pairs aren't as common. + + This filter doesn't convert AUIPC instructions alone because: + + (1) The conversion would be off-by-one (or off-by-4096) half the + time because the lowest 12 bits from inst2 (inst2_imm12) + aren't known. We only know that the absolute address is + pc + AUIPC_imm20 + [-2048, +2047] but there is no way to + know the exact 4096-byte multiple (or 4096 * n + 2048): + there are always two possibilities because AUIPC copies + the 12 lowest bits from pc instead of zeroing them. + + NOTE: The sign-extension of inst2_imm12 adds a tiny bit + of extra complexity to AUIPC math in general but it's not + the reason for this problem. The sign-extension only changes + the relative position of the pc-relative 4096-byte window. + + (2) Matching AUIPC instruction alone requires only seven bits. + When the filter is applied to non-code data, that leads + to many false positives which make compression worse. + As long as most AUIPC+inst2 pairs appear as two consecutive + instructions, converting only such pairs gives better results. + + In assembly, AUIPC+inst2 tend to look like this: + + # Call: + auipc ra, 0x12345 + jalr ra, -42(ra) + + # Tail call: + auipc t1, 0x12345 + jalr zero, -42(t1) + + # Getting the absolute address: + auipc a0, 0x12345 + addi a0, a0, -42 + + # rd of inst2 isn't necessarily the same as rs1 even + # in cases where there is no reason to preserve rs1. + auipc a0, 0x12345 + addi a1, a0, -42 + + As of 2024, 16-bit instructions from the C extension don't + appear as inst2. The RISC-V psABI doesn't list AUIPC+C.* as + a linker relaxation type explicitly but it's not disallowed + either. Usefulness is limited as most of the time the lowest + 12 bits won't fit in a C instruction. This filter doesn't + support AUIPC+C.* combinations because this makes the filter + simpler, there are no test files, and it hopefully will never + be needed anyway. + + (Compare AUIPC to ARM64 where ADRP does set the lowest 12 bits + to zero. The paired instruction has the lowest 12 bits of the + absolute address as is in a zero-extended immediate. Thus the + ARM64 filter doesn't need to care about the instructions that + are paired with ADRP. An off-by-4096 issue can still occur if + the code section isn't aligned with the filter's start offset. + It's not a problem with standalone ELF files but Windows PE + files need start_offset=3072 for best results. Also, a .tar + stores files with 512-byte alignment so most of the time it + won't be the best for ARM64.) + +AUIPC with rd == x0 +------------------- + + AUIPC instructions with rd=x0 are reserved for HINTs in the base + instruction set. Such AUIPC instructions are never filtered. + + As of January 2024, it seems likely that AUIPC with rd=x0 will + be used for landing pads (pseudoinstruction LPAD). LPAD is used + to mark valid targets for indirect jumps (for JALR), for example, + beginnings of functions. The 20-bit immediate in LPAD instruction + is a label, not a pc-relative address. Thus it would be + counterproductive to convert AUIPC instructions with rd=x0. + + Often the next instruction after LPAD won't have rs1=x0 and thus + the filtering would be skipped for that reason alone. However, + it's not good to rely on this. For example, consider a function + that begins like this: + + int foo(int i) + { + if (i <= 234) { + ... + } + + A compiler may generate something like this: + + lpad 0x54321 + li a5, 234 + bgt a0, a5, .L2 + + Converting the pseudoinstructions to raw instructions: + + auipc x0, 0x54321 + addi x15, x0, 234 + blt x15, x10, .L2 + + In this case the filter would undesirably convert the AUIPC+ADDI + pair if the filter didn't explicitly skip AUIPC instructions + that have rd=x0. + +*/ + + +#include "simple_private.h" + + +// This checks two conditions at once: +// - AUIPC rd == inst2 rs1. +// - inst2 opcode has the lowest two bits set. +// +// The 8 bit left shift aligns the rd of AUIPC with the rs1 of inst2. +// By XORing the registers, any non-zero value in those bits indicates the +// registers are not equal and thus not an AUIPC pair. Subtracting 3 from +// inst2 will zero out the first two opcode bits only when they are set. +// The mask tests if any of the register or opcode bits are set (and thus +// not an AUIPC pair). +// +// Alternative expression: (((((auipc) << 8) ^ (inst2)) & 0xF8003) != 3) +#define NOT_AUIPC_PAIR(auipc, inst2) \ + ((((auipc) << 8) ^ ((inst2) - 3)) & 0xF8003) + +// This macro checks multiple conditions: +// (1) AUIPC rd [11:7] == x2 (special rd value). +// (2) AUIPC bits 12 and 13 set (the lowest two opcode bits of packed inst2). +// (3) inst2_rs1 doesn't equal x0 or x2 because the opposite +// conversion is only done when +// auipc_rd != x0 && +// auipc_rd != x2 && +// auipc_rd == inst2_rs1. +// +// The left-hand side takes care of (1) and (2). +// (a) The lowest 7 bits are already known to be AUIPC so subtracting 0x17 +// makes those bits zeros. +// (b) If AUIPC rd equals x2, subtracting 0x100 makes bits [11:7] zeros. +// If rd doesn't equal x2, then there will be at least one non-zero bit +// and the next step (c) is irrelevant. +// (c) If the lowest two opcode bits of the packed inst2 are set in [13:12], +// then subtracting 0x3000 will make those bits zeros. Otherwise there +// will be at least one non-zero bit. +// +// The shift by 18 removes the high bits from the final '>=' comparison and +// ensures that any non-zero result will be larger than any possible result +// from the right-hand side of the comparison. The cast ensures that the +// left-hand side didn't get promoted to a larger type than uint32_t. +// +// On the right-hand side, inst2_rs1 & 0x1D will be non-zero as long as +// inst2_rs1 is not x0 or x2. +// +// The final '>=' comparison will make the expression true if: +// - The subtraction caused any bits to be set (special AUIPC rd value not +// used or inst2 opcode bits not set). (non-zero >= non-zero or 0) +// - The subtraction did not cause any bits to be set but inst2_rs1 was +// x0 or x2. (0 >= 0) +#define NOT_SPECIAL_AUIPC(auipc, inst2_rs1) \ + ((uint32_t)(((auipc) - 0x3117) << 18) >= ((inst2_rs1) & 0x1D)) + + +// The encode and decode functions are split for this filter because of the +// AUIPC+inst2 filtering. This filter design allows a decoder-only +// implementation to be smaller than alternative designs. + +#ifdef HAVE_ENCODER_RISCV +static size_t +riscv_encode(void *simple lzma_attribute((__unused__)), + uint32_t now_pos, + bool is_encoder lzma_attribute((__unused__)), + uint8_t *buffer, size_t size) +{ + // Avoid using i + 8 <= size in the loop condition. + // + // NOTE: If there is a JAL in the last six bytes of the stream, it + // won't be converted. This is intentional to keep the code simpler. + if (size < 8) + return 0; + + size -= 8; + + size_t i; + + // The loop is advanced by 2 bytes every iteration since the + // instruction stream may include 16-bit instructions (C extension). + for (i = 0; i <= size; i += 2) { + uint32_t inst = buffer[i]; + + if (inst == 0xEF) { + // JAL + const uint32_t b1 = buffer[i + 1]; + + // Only filter rd=x1(ra) and rd=x5(t0). + if ((b1 & 0x0D) != 0) + continue; + + // The 20-bit immediate is in four pieces. + // The encoder stores it in big endian form + // since it improves compression slightly. + const uint32_t b2 = buffer[i + 2]; + const uint32_t b3 = buffer[i + 3]; + const uint32_t pc = now_pos + (uint32_t)i; + +// The following chart shows the highest three bytes of JAL, focusing on +// the 20-bit immediate field [31:12]. The first row of numbers is the +// bit position in a 32-bit little endian instruction. The second row of +// numbers shows the order of the immediate field in a J-type instruction. +// The last row is the bit number in each byte. +// +// To determine the amount to shift each bit, subtract the value in +// the last row from the value in the second last row. If the number +// is positive, shift left. If negative, shift right. +// +// For example, at the rightmost side of the chart, the bit 4 in b1 is +// the bit 12 of the address. Thus that bit needs to be shifted left +// by 12 - 4 = 8 bits to put it in the right place in the addr variable. +// +// NOTE: The immediate of a J-type instruction holds bits [20:1] of +// the address. The bit [0] is always 0 and not part of the immediate. +// +// | b3 | b2 | b1 | +// | 31 30 29 28 27 26 25 24 | 23 22 21 20 19 18 17 16 | 15 14 13 12 x x x x | +// | 20 10 9 8 7 6 5 4 | 3 2 1 11 19 18 17 16 | 15 14 13 12 x x x x | +// | 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | 7 6 5 4 x x x x | + + uint32_t addr = ((b1 & 0xF0) << 8) + | ((b2 & 0x0F) << 16) + | ((b2 & 0x10) << 7) + | ((b2 & 0xE0) >> 4) + | ((b3 & 0x7F) << 4) + | ((b3 & 0x80) << 13); + + addr += pc; + + buffer[i + 1] = (uint8_t)((b1 & 0x0F) + | ((addr >> 13) & 0xF0)); + + buffer[i + 2] = (uint8_t)(addr >> 9); + buffer[i + 3] = (uint8_t)(addr >> 1); + + // The "-2" is included because the for-loop will + // always increment by 2. In this case, we want to + // skip an extra 2 bytes since we used 4 bytes + // of input. + i += 4 - 2; + + } else if ((inst & 0x7F) == 0x17) { + // AUIPC + inst |= (uint32_t)buffer[i + 1] << 8; + inst |= (uint32_t)buffer[i + 2] << 16; + inst |= (uint32_t)buffer[i + 3] << 24; + + // Branch based on AUIPC's rd. The bitmask test does + // the same thing as this: + // + // const uint32_t auipc_rd = (inst >> 7) & 0x1F; + // if (auipc_rd != 0 && auipc_rd != 2) { + if (inst & 0xE80) { + // AUIPC's rd doesn't equal x0 or x2. + + // Check if AUIPC+inst2 are a pair. + uint32_t inst2 = read32le(buffer + i + 4); + + if (NOT_AUIPC_PAIR(inst, inst2)) { + // The NOT_AUIPC_PAIR macro allows + // a false AUIPC+AUIPC pair if the + // bits [19:15] (where rs1 would be) + // in the second AUIPC match the rd + // of the first AUIPC. + // + // We must skip enough forward so + // that the first two bytes of the + // second AUIPC cannot get converted. + // Such a conversion could make the + // current pair become a valid pair + // which would desync the decoder. + // + // Skipping six bytes is enough even + // though the above condition looks + // at the lowest four bits of the + // buffer[i + 6] too. This is safe + // because this filter never changes + // those bits if a conversion at + // that position is done. + i += 6 - 2; + continue; + } + + // Convert AUIPC+inst2 to a special format: + // + // - The lowest 7 bits [6:0] retain the + // AUIPC opcode. + // + // - The rd [11:7] is set to x2(sp). x2 is + // used as the stack pointer so AUIPC with + // rd=x2 should be very rare in real-world + // executables. + // + // - The remaining 20 bits [31:12] (that + // normally hold the pc-relative immediate) + // are used to store the lowest 20 bits of + // inst2. That is, the 12-bit immediate of + // inst2 is not included. + // + // - The location of the original inst2 is + // used to store the 32-bit absolute + // address in big endian format. Compared + // to the 20+12-bit split encoding, this + // results in a longer uninterrupted + // sequence of identical common bytes + // when the same address is referred + // with different instruction pairs + // (like AUIPC+LD vs. AUIPC+ADDI) or + // when the occurrences of the same + // pair use different registers. When + // referring to adjacent memory locations + // (like function calls that go via the + // ELF PLT), in big endian order only the + // last 1-2 bytes differ; in little endian + // the differing 1-2 bytes would be in the + // middle of the 8-byte sequence. + // + // When reversing the transformation, the + // original rd of AUIPC can be restored + // from inst2's rs1 as they are required to + // be the same. + + // Arithmetic right shift makes sign extension + // trivial but (1) it's implementation-defined + // behavior (C99/C11/C23 6.5.7-p5) and so is + // (2) casting unsigned to signed (6.3.1.3-p3). + // + // One can check for (1) with + // + // if ((-1 >> 1) == -1) ... + // + // but (2) has to be checked from the + // compiler docs. GCC promises that (1) + // and (2) behave in the common expected + // way and thus + // + // addr += (uint32_t)( + // (int32_t)inst2 >> 20); + // + // does the same as the code below. But since + // the 100 % portable way is only a few bytes + // bigger code and there is no real speed + // difference, let's just use that, especially + // since the decoder doesn't need this at all. + uint32_t addr = inst & 0xFFFFF000; + addr += (inst2 >> 20) + - ((inst2 >> 19) & 0x1000); + + addr += now_pos + (uint32_t)i; + + // Construct the first 32 bits: + // [6:0] AUIPC opcode + // [11:7] Special AUIPC rd = x2 + // [31:12] The lowest 20 bits of inst2 + inst = 0x17 | (2 << 7) | (inst2 << 12); + + write32le(buffer + i, inst); + + // The second 32 bits store the absolute + // address in big endian order. + write32be(buffer + i + 4, addr); + } else { + // AUIPC's rd equals x0 or x2. + // + // x0 indicates a landing pad (LPAD). + // It's always skipped. + // + // AUIPC with rd == x2 is used for the special + // format as explained above. When the input + // contains a byte sequence that matches the + // special format, "fake" decoding must be + // done to keep the filter bijective (that + // is, safe to apply on arbitrary data). + // + // See the "x0 or x2" section in riscv_decode() + // for how the "real" decoding is done. The + // "fake" decoding is a simplified version + // of "real" decoding with the following + // differences (these reduce code size of + // the decoder): + // (1) The lowest 12 bits aren't sign-extended. + // (2) No address conversion is done. + // (3) Big endian format isn't used (the fake + // address is in little endian order). + + // Check if inst matches the special format. + const uint32_t fake_rs1 = inst >> 27; + + if (NOT_SPECIAL_AUIPC(inst, fake_rs1)) { + i += 4 - 2; + continue; + } + + const uint32_t fake_addr = + read32le(buffer + i + 4); + + // Construct the second 32 bits: + // [19:0] Upper 20 bits from AUIPC + // [31:20] The lowest 12 bits of fake_addr + const uint32_t fake_inst2 = (inst >> 12) + | (fake_addr << 20); + + // Construct new first 32 bits from: + // [6:0] AUIPC opcode + // [11:7] Fake AUIPC rd = fake_rs1 + // [31:12] The highest 20 bits of fake_addr + inst = 0x17 | (fake_rs1 << 7) + | (fake_addr & 0xFFFFF000); + + write32le(buffer + i, inst); + write32le(buffer + i + 4, fake_inst2); + } + + i += 8 - 2; + } + } + + return i; +} + + +extern lzma_ret +lzma_simple_riscv_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return lzma_simple_coder_init(next, allocator, filters, + &riscv_encode, 0, 8, 2, true); +} +#endif + + +#ifdef HAVE_DECODER_RISCV +static size_t +riscv_decode(void *simple lzma_attribute((__unused__)), + uint32_t now_pos, + bool is_encoder lzma_attribute((__unused__)), + uint8_t *buffer, size_t size) +{ + if (size < 8) + return 0; + + size -= 8; + + size_t i; + for (i = 0; i <= size; i += 2) { + uint32_t inst = buffer[i]; + + if (inst == 0xEF) { + // JAL + const uint32_t b1 = buffer[i + 1]; + + // Only filter rd=x1(ra) and rd=x5(t0). + if ((b1 & 0x0D) != 0) + continue; + + const uint32_t b2 = buffer[i + 2]; + const uint32_t b3 = buffer[i + 3]; + const uint32_t pc = now_pos + (uint32_t)i; + +// | b3 | b2 | b1 | +// | 31 30 29 28 27 26 25 24 | 23 22 21 20 19 18 17 16 | 15 14 13 12 x x x x | +// | 20 10 9 8 7 6 5 4 | 3 2 1 11 19 18 17 16 | 15 14 13 12 x x x x | +// | 7 6 5 4 3 2 1 0 | 7 6 5 4 3 2 1 0 | 7 6 5 4 x x x x | + + uint32_t addr = ((b1 & 0xF0) << 13) + | (b2 << 9) | (b3 << 1); + + addr -= pc; + + buffer[i + 1] = (uint8_t)((b1 & 0x0F) + | ((addr >> 8) & 0xF0)); + + buffer[i + 2] = (uint8_t)(((addr >> 16) & 0x0F) + | ((addr >> 7) & 0x10) + | ((addr << 4) & 0xE0)); + + buffer[i + 3] = (uint8_t)(((addr >> 4) & 0x7F) + | ((addr >> 13) & 0x80)); + + i += 4 - 2; + + } else if ((inst & 0x7F) == 0x17) { + // AUIPC + uint32_t inst2; + + inst |= (uint32_t)buffer[i + 1] << 8; + inst |= (uint32_t)buffer[i + 2] << 16; + inst |= (uint32_t)buffer[i + 3] << 24; + + if (inst & 0xE80) { + // AUIPC's rd doesn't equal x0 or x2. + + // Check if it is a "fake" AUIPC+inst2 pair. + inst2 = read32le(buffer + i + 4); + + if (NOT_AUIPC_PAIR(inst, inst2)) { + i += 6 - 2; + continue; + } + + // Decode (or more like re-encode) the "fake" + // pair. The "fake" format doesn't do + // sign-extension, address conversion, or + // use big endian. (The use of little endian + // allows sharing the write32le() calls in + // the decoder to reduce code size when + // unaligned access isn't supported.) + uint32_t addr = inst & 0xFFFFF000; + addr += inst2 >> 20; + + inst = 0x17 | (2 << 7) | (inst2 << 12); + inst2 = addr; + } else { + // AUIPC's rd equals x0 or x2. + + // Check if inst matches the special format + // used by the encoder. + const uint32_t inst2_rs1 = inst >> 27; + + if (NOT_SPECIAL_AUIPC(inst, inst2_rs1)) { + i += 4 - 2; + continue; + } + + // Decode the "real" pair. + uint32_t addr = read32be(buffer + i + 4); + + addr -= now_pos + (uint32_t)i; + + // The second instruction: + // - Get the lowest 20 bits from inst. + // - Add the lowest 12 bits of the address + // as the immediate field. + inst2 = (inst >> 12) | (addr << 20); + + // AUIPC: + // - rd is the same as inst2_rs1. + // - The sign extension of the lowest 12 bits + // must be taken into account. + inst = 0x17 | (inst2_rs1 << 7) + | ((addr + 0x800) & 0xFFFFF000); + } + + // Both decoder branches write in little endian order. + write32le(buffer + i, inst); + write32le(buffer + i + 4, inst2); + + i += 8 - 2; + } + } + + return i; +} + + +extern lzma_ret +lzma_simple_riscv_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters) +{ + return lzma_simple_coder_init(next, allocator, filters, + &riscv_decode, 0, 8, 2, false); +} +#endif diff --git a/src/liblzma/simple/simple_coder.c b/src/liblzma/simple/simple_coder.c index ed2d7fb..5cbfa82 100644 --- a/src/liblzma/simple/simple_coder.c +++ b/src/liblzma/simple/simple_coder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_coder.c @@ -8,9 +10,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/simple_coder.h b/src/liblzma/simple/simple_coder.h index 668a509..2b762d5 100644 --- a/src/liblzma/simple/simple_coder.h +++ b/src/liblzma/simple/simple_coder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_coder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_CODER_H @@ -62,12 +61,12 @@ extern lzma_ret lzma_simple_armthumb_decoder_init(lzma_next_coder *next, extern lzma_ret lzma_simple_arm64_encoder_init(lzma_next_coder *next, - const lzma_allocator *allocator, - const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_arm64_decoder_init(lzma_next_coder *next, - const lzma_allocator *allocator, - const lzma_filter_info *filters); + const lzma_allocator *allocator, + const lzma_filter_info *filters); extern lzma_ret lzma_simple_sparc_encoder_init(lzma_next_coder *next, @@ -78,4 +77,13 @@ extern lzma_ret lzma_simple_sparc_decoder_init(lzma_next_coder *next, const lzma_allocator *allocator, const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_riscv_encoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters); + +extern lzma_ret lzma_simple_riscv_decoder_init(lzma_next_coder *next, + const lzma_allocator *allocator, + const lzma_filter_info *filters); + #endif diff --git a/src/liblzma/simple/simple_decoder.c b/src/liblzma/simple/simple_decoder.c index dc4d241..d9820ee 100644 --- a/src/liblzma/simple/simple_decoder.c +++ b/src/liblzma/simple/simple_decoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_decoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_decoder.h" diff --git a/src/liblzma/simple/simple_decoder.h b/src/liblzma/simple/simple_decoder.h index bed8d37..2ae87bb 100644 --- a/src/liblzma/simple/simple_decoder.h +++ b/src/liblzma/simple/simple_decoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_decoder.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_DECODER_H diff --git a/src/liblzma/simple/simple_encoder.c b/src/liblzma/simple/simple_encoder.c index d2cc03e..d1f3509 100644 --- a/src/liblzma/simple/simple_encoder.c +++ b/src/liblzma/simple/simple_encoder.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_encoder.h" diff --git a/src/liblzma/simple/simple_encoder.h b/src/liblzma/simple/simple_encoder.h index 1cee482..bf5edbb 100644 --- a/src/liblzma/simple/simple_encoder.h +++ b/src/liblzma/simple/simple_encoder.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_encoder.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_ENCODER_H diff --git a/src/liblzma/simple/simple_private.h b/src/liblzma/simple/simple_private.h index 9d2c0fd..7aa360f 100644 --- a/src/liblzma/simple/simple_private.h +++ b/src/liblzma/simple/simple_private.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file simple_private.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #ifndef LZMA_SIMPLE_PRIVATE_H diff --git a/src/liblzma/simple/sparc.c b/src/liblzma/simple/sparc.c index bad8492..e8ad285 100644 --- a/src/liblzma/simple/sparc.c +++ b/src/liblzma/simple/sparc.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file sparc.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" diff --git a/src/liblzma/simple/x86.c b/src/liblzma/simple/x86.c index 232b295..f216231 100644 --- a/src/liblzma/simple/x86.c +++ b/src/liblzma/simple/x86.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file x86.c @@ -6,9 +8,6 @@ // Authors: Igor Pavlov // Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "simple_private.h" @@ -27,11 +26,7 @@ static size_t x86_code(void *simple_ptr, uint32_t now_pos, bool is_encoder, uint8_t *buffer, size_t size) { - static const bool MASK_TO_ALLOWED_STATUS[8] - = { true, true, true, false, true, false, false, false }; - - static const uint32_t MASK_TO_BIT_NUMBER[8] - = { 0, 1, 2, 2, 3, 3, 3, 3 }; + static const uint32_t MASK_TO_BIT_NUMBER[5] = { 0, 1, 2, 2, 3 }; lzma_simple_x86 *simple = simple_ptr; uint32_t prev_mask = simple->prev_mask; @@ -68,9 +63,8 @@ x86_code(void *simple_ptr, uint32_t now_pos, bool is_encoder, b = buffer[buffer_pos + 4]; - if (Test86MSByte(b) - && MASK_TO_ALLOWED_STATUS[(prev_mask >> 1) & 0x7] - && (prev_mask >> 1) < 0x10) { + if (Test86MSByte(b) && (prev_mask >> 1) <= 4 + && (prev_mask >> 1) != 3) { uint32_t src = ((uint32_t)(b) << 24) | ((uint32_t)(buffer[buffer_pos + 3]) << 16) diff --git a/src/liblzma/validate_map.sh b/src/liblzma/validate_map.sh index 2bf6f8b..dd1589d 100644 --- a/src/liblzma/validate_map.sh +++ b/src/liblzma/validate_map.sh @@ -1,4 +1,5 @@ #!/bin/sh +# SPDX-License-Identifier: 0BSD ############################################################################### # @@ -78,9 +79,6 @@ # # Author: Lasse Collin # -# This file has been put into the public domain. -# You can do whatever you want with this file. -# ############################################################################### LC_ALL=C @@ -124,7 +122,7 @@ DUPS=$(sort liblzma_generic.map | sed '/^$/d;/^global:$/d' | uniq -d) # ignored (@XZ_5.1.2alpha or @XZ_5.2.2 won't be added at all when # the #define HAVE_SYMBOL_VERSIONS_LINUX isn't used). IN_SYNC= -if ! sed '109,123d' liblzma_linux.map \ +if ! sed '111,125d' liblzma_linux.map \ | cmp -s - liblzma_generic.map; then IN_SYNC=no fi diff --git a/src/lzmainfo/Makefile.am b/src/lzmainfo/Makefile.am index ff7172b..9a5bc68 100644 --- a/src/lzmainfo/Makefile.am +++ b/src/lzmainfo/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## bin_PROGRAMS = lzmainfo @@ -37,3 +33,29 @@ dist_man_MANS = lzmainfo.1 .rc.o: $(RC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ $(lzmainfo_CPPFLAGS) $(CPPFLAGS) $(RCFLAGS) -i $< -o $@ + +# The installation of translated man pages abuses Automake internals +# by calling "install-man" with redefined dist_man_MANS and man_MANS. +# If this breaks some day, don't blame Automake developers. +install-data-hook: + languages= ; \ + if test "$(USE_NLS)" = yes && test -d "$(top_srcdir)/po4a/man"; then \ + languages=`ls "$(top_srcdir)/po4a/man"`; \ + fi; \ + for lang in . $$languages; do \ + man="$(top_srcdir)/po4a/man/$$lang/lzmainfo.1" ; \ + if test -f "$$man"; then \ + $(MAKE) dist_man_MANS="$$man" man_MANS= \ + mandir="$(mandir)/$$lang" install-man; \ + fi; \ + done + +uninstall-hook: + languages= ; \ + if test "$(USE_NLS)" = yes && test -d "$(top_srcdir)/po4a/man"; then \ + languages=`ls "$(top_srcdir)/po4a/man"`; \ + fi; \ + for lang in . $$languages; do \ + name=`echo lzmainfo | sed '$(transform)'` && \ + rm -f "$(DESTDIR)$(mandir)/$$lang/man1/$$name.1"; \ + done diff --git a/src/lzmainfo/Makefile.in b/src/lzmainfo/Makefile.in index 0a18942..feea983 100644 --- a/src/lzmainfo/Makefile.in +++ b/src/lzmainfo/Makefile.in @@ -93,8 +93,8 @@ bin_PROGRAMS = lzmainfo$(EXEEXT) @COND_GNULIB_TRUE@am__append_2 = $(top_builddir)/lib/libgnu.a subdir = src/lzmainfo ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -237,7 +237,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -366,6 +365,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ @@ -756,7 +757,8 @@ info: info-am info-am: install-data-am: install-man - + @$(NORMAL_INSTALL) + $(MAKE) $(AM_MAKEFLAGS) install-data-hook install-dvi: install-dvi-am install-dvi-am: @@ -804,25 +806,27 @@ ps: ps-am ps-am: uninstall-am: uninstall-binPROGRAMS uninstall-man - + @$(NORMAL_INSTALL) + $(MAKE) $(AM_MAKEFLAGS) uninstall-hook uninstall-man: uninstall-man1 -.MAKE: install-am install-strip +.MAKE: install-am install-data-am install-strip uninstall-am .PHONY: CTAGS GTAGS TAGS all all-am am--depfiles check check-am clean \ clean-binPROGRAMS clean-generic clean-libtool cscopelist-am \ ctags ctags-am distclean distclean-compile distclean-generic \ distclean-libtool distclean-tags distdir dvi dvi-am html \ html-am info info-am install install-am install-binPROGRAMS \ - install-data install-data-am install-dvi install-dvi-am \ - install-exec install-exec-am install-html install-html-am \ - install-info install-info-am install-man install-man1 \ - install-pdf install-pdf-am install-ps install-ps-am \ - install-strip installcheck installcheck-am installdirs \ - maintainer-clean maintainer-clean-generic mostlyclean \ - mostlyclean-compile mostlyclean-generic mostlyclean-libtool \ - pdf pdf-am ps ps-am tags tags-am uninstall uninstall-am \ - uninstall-binPROGRAMS uninstall-man uninstall-man1 + install-data install-data-am install-data-hook install-dvi \ + install-dvi-am install-exec install-exec-am install-html \ + install-html-am install-info install-info-am install-man \ + install-man1 install-pdf install-pdf-am install-ps \ + install-ps-am install-strip installcheck installcheck-am \ + installdirs maintainer-clean maintainer-clean-generic \ + mostlyclean mostlyclean-compile mostlyclean-generic \ + mostlyclean-libtool pdf pdf-am ps ps-am tags tags-am uninstall \ + uninstall-am uninstall-binPROGRAMS uninstall-hook \ + uninstall-man uninstall-man1 .PRECIOUS: Makefile @@ -832,6 +836,32 @@ uninstall-man: uninstall-man1 $(RC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) \ $(lzmainfo_CPPFLAGS) $(CPPFLAGS) $(RCFLAGS) -i $< -o $@ +# The installation of translated man pages abuses Automake internals +# by calling "install-man" with redefined dist_man_MANS and man_MANS. +# If this breaks some day, don't blame Automake developers. +install-data-hook: + languages= ; \ + if test "$(USE_NLS)" = yes && test -d "$(top_srcdir)/po4a/man"; then \ + languages=`ls "$(top_srcdir)/po4a/man"`; \ + fi; \ + for lang in . $$languages; do \ + man="$(top_srcdir)/po4a/man/$$lang/lzmainfo.1" ; \ + if test -f "$$man"; then \ + $(MAKE) dist_man_MANS="$$man" man_MANS= \ + mandir="$(mandir)/$$lang" install-man; \ + fi; \ + done + +uninstall-hook: + languages= ; \ + if test "$(USE_NLS)" = yes && test -d "$(top_srcdir)/po4a/man"; then \ + languages=`ls "$(top_srcdir)/po4a/man"`; \ + fi; \ + for lang in . $$languages; do \ + name=`echo lzmainfo | sed '$(transform)'` && \ + rm -f "$(DESTDIR)$(mandir)/$$lang/man1/$$name.1"; \ + done + # Tell versions [3.59,3.63) of GNU make to not export all variables. # Otherwise a system limit (for SysV at least) may be exceeded. .NOEXPORT: diff --git a/src/lzmainfo/lzmainfo.1 b/src/lzmainfo/lzmainfo.1 index ce38eee..114e0f3 100644 --- a/src/lzmainfo/lzmainfo.1 +++ b/src/lzmainfo/lzmainfo.1 @@ -1,9 +1,7 @@ +.\" SPDX-License-Identifier: 0BSD .\" .\" Author: Lasse Collin .\" -.\" This file has been put into the public domain. -.\" You can do whatever you want with this file. -.\" .TH LZMAINFO 1 "2013-06-30" "Tukaani" "XZ Utils" .SH NAME lzmainfo \- show information stored in the .lzma file header diff --git a/src/lzmainfo/lzmainfo.c b/src/lzmainfo/lzmainfo.c index 71e6295..2550b1f 100644 --- a/src/lzmainfo/lzmainfo.c +++ b/src/lzmainfo/lzmainfo.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file lzmainfo.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "sysdefs.h" diff --git a/src/lzmainfo/lzmainfo_w32res.rc b/src/lzmainfo/lzmainfo_w32res.rc index 378fbe6..f2ec288 100644 --- a/src/lzmainfo/lzmainfo_w32res.rc +++ b/src/lzmainfo/lzmainfo_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_APP diff --git a/src/scripts/Makefile.am b/src/scripts/Makefile.am index fe5742d..29c561c 100644 --- a/src/scripts/Makefile.am +++ b/src/scripts/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## nodist_bin_SCRIPTS = xzdiff xzgrep xzmore xzless dist_man_MANS = xzdiff.1 xzgrep.1 xzmore.1 xzless.1 diff --git a/src/scripts/Makefile.in b/src/scripts/Makefile.in index 42dbfff..5cd5dbd 100644 --- a/src/scripts/Makefile.in +++ b/src/scripts/Makefile.in @@ -99,8 +99,8 @@ host_triplet = @host@ subdir = src/scripts ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -189,7 +189,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -318,6 +317,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ diff --git a/src/scripts/xzdiff.1 b/src/scripts/xzdiff.1 index 24fed68..8f473ed 100644 --- a/src/scripts/xzdiff.1 +++ b/src/scripts/xzdiff.1 @@ -1,67 +1,90 @@ +.\" SPDX-License-Identifier: 0BSD .\" -.\" Original zdiff.1 for gzip: Jean-loup Gailly +.\" Authors: Lasse Collin +.\" Jia Tan .\" -.\" Modifications for XZ Utils: Lasse Collin -.\" Andrew Dudman +.\" (Note that this file is not based on gzip's zdiff.1.) .\" -.\" License: GNU GPLv2+ -.\" -.TH XZDIFF 1 "2021-06-04" "Tukaani" "XZ Utils" +.TH XZDIFF 1 "2024-02-13" "Tukaani" "XZ Utils" .SH NAME xzcmp, xzdiff, lzcmp, lzdiff \- compare compressed files +. .SH SYNOPSIS .B xzcmp -.RI [ cmp_options "] " file1 " [" file2 ] +.RI [ option... ] +.I file1 +.RI [ file2 ] .br .B xzdiff -.RI [ diff_options "] " file1 " [" file2 ] +\&... .br .B lzcmp -.RI [ cmp_options "] " file1 " [" file2 ] +\&... .br .B lzdiff -.RI [ diff_options "] " file1 " [" file2 ] +\&... +. .SH DESCRIPTION .B xzcmp and .B xzdiff -invoke +compare uncompressed contents of two files. +Uncompressed data and options are passed to .BR cmp (1) or .BR diff (1) -on files compressed with +unless +.B \-\-help +or +.B \-\-version +is specified. +.PP +If both +.I file1 +and +.I file2 +are specified, they can be uncompressed files or files in formats that .BR xz (1), -.BR lzma (1), .BR gzip (1), .BR bzip2 (1), .BR lzop (1), +.BR zstd (1), or -.BR zstd (1). -All options specified are passed directly to -.BR cmp (1) -or -.BR diff (1). -If only one file is specified, then the files compared are +.BR lz4 (1) +can decompress. +The required decompression commands are determined from +the filename suffixes of .I file1 -(which must have a suffix of a supported compression format) and +and +.IR file2 . +A file with an unknown suffix is assumed to be either uncompressed +or in a format that +.BR xz (1) +can decompress. +.PP +If only one filename is provided, .I file1 -from which the compression format suffix has been stripped. -If two files are specified, -then they are uncompressed if necessary and fed to -.BR cmp (1) -or -.BR diff (1). -The exit status from -.BR cmp (1) -or -.BR diff (1) -is preserved unless a decompression error occurs; then exit status is 2. +must have a suffix of a supported compression format and the name for +.I file2 +is assumed to be +.I file1 +with the compression format suffix removed. .PP -The names +The commands .B lzcmp and .B lzdiff are provided for backward compatibility with LZMA Utils. +. +.SH EXIT STATUS +If a decompression error occurs, the exit status is +.BR 2 . +Otherwise the exit status of +.BR cmp (1) +or +.BR diff (1) +is used. +. .SH "SEE ALSO" .BR cmp (1), .BR diff (1), @@ -70,10 +93,4 @@ are provided for backward compatibility with LZMA Utils. .BR bzip2 (1), .BR lzop (1), .BR zstd (1), -.BR zdiff (1) -.SH BUGS -Messages from the -.BR cmp (1) -or -.BR diff (1) -programs refer to temporary filenames instead of those specified. +.BR lz4 (1) diff --git a/src/scripts/xzdiff.in b/src/scripts/xzdiff.in index c17d78b..179218d 100644 --- a/src/scripts/xzdiff.in +++ b/src/scripts/xzdiff.in @@ -1,4 +1,5 @@ #!@POSIX_SHELL@ +# SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 1998, 2002, 2006, 2007 Free Software Foundation # Copyright (C) 1993 Jean-loup Gailly @@ -30,7 +31,7 @@ case ${0##*/} in *) prog=xzdiff; cmp=${DIFF:-diff};; esac -version="$prog (@PACKAGE_NAME@) @VERSION@" +version="$prog (@PACKAGE_NAME@) @PACKAGE_VERSION@" usage="Usage: ${0##*/} [OPTION]... FILE1 [FILE2] Compare FILE1 to FILE2, using their uncompressed contents if they are @@ -83,13 +84,15 @@ if test $# -eq 1; then xz1=lzop;; *[-.]zst | *.tzst) xz1='zstd -q';; + *[-.]lz4) + xz1=lz4;; *) printf '%s\n' "$0: $1: Unknown compressed file name suffix" >&2 exit 2;; esac case $1 in - *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *[-.]lz | *[-.]lzo | *[-.]zst) - FILE=`expr "X$1" : 'X\(.*\)[-.][abglmostxzZ2]*$'`;; + *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *[-.]lz | *[-.]lzo | *[-.]zst | *[-.]lz4) + FILE=`expr "X$1" : 'X\(.*\)[-.][abglmostxzZ24]*$'`;; *.t[abglx]z) FILE=`expr "X$1" : 'X\(.*[-.]t\)[abglx]z$'`ar;; *.tbz2) @@ -109,17 +112,19 @@ elif test $# -eq 2; then *[-.][zZ] | *_z | *[-.]gz | *.t[ag]z) xz1=gzip;; *[-.]lzo | *.tzo) xz1=lzop;; *[-.]zst | *.tzst) xz1='zstd -q';; + *[-.]lz4) xz1=lz4;; esac case $2 in *[-.]bz2 | *.tbz | *.tbz2) xz2=bzip2;; *[-.][zZ] | *_z | *[-.]gz | *.t[ag]z) xz2=gzip;; *[-.]lzo | *.tzo) xz2=lzop;; *[-.]zst | *.tzst) xz2='zstd -q';; + *[-.]lz4) xz2=lz4;; esac case $1 in - *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *[-.]lz | *.t[abglx]z | *.tbz2 | *[-.]lzo | *.tzo | *[-.]zst | *.tzst | -) + *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *[-.]lz | *.t[abglx]z | *.tbz2 | *[-.]lzo | *.tzo | *[-.]zst | *.tzst | *[-.]lz4 | -) case "$2" in - *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *[-.]lz | *.t[abglx]z | *.tbz2 | *[-.]lzo | *.tzo | *[-.]zst | *.tzst | -) + *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *[-.]lz | *.t[abglx]z | *.tbz2 | *[-.]lzo | *.tzo | *[-.]zst | *.tzst | *[-.]lz4 | -) if test "$1$2" = --; then xz_status=$( exec 4>&1 @@ -183,7 +188,7 @@ elif test $# -eq 2; then esac;; *) case "$2" in - *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *[-.]lz | *.t[abglx]z | *.tbz2 | *[-.]lzo | *.tzo | *[-.]zst | *.tzst | -) + *[-.][zZ] | *_z | *[-.][gx]z | *[-.]bz2 | *[-.]lzma | *[-.]lz | *.t[abglx]z | *.tbz2 | *[-.]lzo | *.tzo | *[-.]zst | *.tzst | *[-.]lz4 | -) xz_status=$( exec 4>&1 ($xz2 -cdf -- "$2" 4>&-; echo $? >&4) 3>&- | @@ -194,7 +199,7 @@ elif test $# -eq 2; then esac;; esac else - printf '%s\n' "$0: Invalid number of operands; try \`${0##*/} --help' for help" >&2 + printf '%s\n' "$0: Invalid number of operands; try '${0##*/} --help' for help" >&2 exit 2 fi diff --git a/src/scripts/xzgrep.1 b/src/scripts/xzgrep.1 index e0514b8..63f012c 100644 --- a/src/scripts/xzgrep.1 +++ b/src/scripts/xzgrep.1 @@ -1,19 +1,18 @@ +.\" SPDX-License-Identifier: 0BSD .\" -.\" Original zgrep.1 for gzip: Jean-loup Gailly -.\" Charles Levert <charles@comm.polymtl.ca> +.\" Authors: Lasse Collin +.\" Jia Tan .\" -.\" Modifications for XZ Utils: Lasse Collin +.\" (Note that this file is not based on gzip's zgrep.1.) .\" -.\" License: GNU GPLv2+ -.\" -.TH XZGREP 1 "2022-07-19" "Tukaani" "XZ Utils" +.TH XZGREP 1 "2024-02-13" "Tukaani" "XZ Utils" .SH NAME -xzgrep \- search compressed files for a regular expression +xzgrep \- search possibly-compressed files for patterns +. .SH SYNOPSIS .B xzgrep -.RI [ grep_options ] -.RB [ \-e ] -.I pattern +.RI [ option... ] +.RI [ pattern_list ] .RI [ file... ] .br .B xzegrep @@ -30,54 +29,84 @@ xzgrep \- search compressed files for a regular expression .br .B lzfgrep \&... +. .SH DESCRIPTION .B xzgrep invokes .BR grep (1) -on +on uncompressed contents of files. +The formats of the .I files -which may be either uncompressed or compressed with +are determined from the filename suffixes. +Any +.I file +with a suffix supported by .BR xz (1), -.BR lzma (1), .BR gzip (1), .BR bzip2 (1), .BR lzop (1), +.BR zstd (1), or -.BR zstd (1). -All options specified are passed directly to -.BR grep (1). +.BR lz4 (1) +will be decompressed; +all other files are assumed to be uncompressed. .PP If no +.I files +are specified or .I file -is specified, then standard input is decompressed if necessary -and fed to -.BR grep (1). -When reading from standard input, -.BR gzip (1), -.BR bzip2 (1), -.BR lzop (1), -and -.BR zstd (1) -compressed files are not supported. +is +.B \- +then standard input is read. +When reading from standard input, only files supported by +.BR xz (1) +are decompressed. +Other files are assumed to be in uncompressed form already. +.PP +Most +.I options +of +.BR grep (1) +are supported. +However, the following options are not supported: +.IP "" 4 +.BR \-r , +.B \-\-recursive +.IP "" 4 +.BR \-R , +.B \-\-dereference\-recursive +.IP "" 4 +.BR \-d , +.BI \-\-directories= action +.IP "" 4 +.BR \-Z , +.B \-\-null +.IP "" 4 +.BR \-z , +.B \-\-null\-data +.IP "" 4 +.BI \-\-include= glob +.IP "" 4 +.BI \-\-exclude= glob +.IP "" 4 +.BI \-\-exclude\-from= file +.IP "" 4 +.BI \-\-exclude\-dir= glob .PP -If -.B xzgrep -is invoked as .B xzegrep -or +is an alias for +.BR "xzgrep \-E" . .B xzfgrep -then -.B grep \-E -or -.B grep \-F -is used instead of -.BR grep (1). -The same applies to names +is an alias for +.BR "xzgrep \-F" . +.PP +The commands .BR lzgrep , .BR lzegrep , and -.BR lzfgrep , -which are provided for backward compatibility with LZMA Utils. +.B lzfgrep +are provided for backward compatibility with LZMA Utils. +. .SH EXIT STATUS .TP 0 @@ -91,18 +120,19 @@ No errors occurred. >1 One or more errors occurred. It is unknown if matches were found. +. .SH ENVIRONMENT .TP .B GREP -If the +If .B GREP -environment variable is set, -.B xzgrep -uses it instead of -.BR grep (1), +is set to a non-empty value, +it is used instead of +.BR "grep" , .BR "grep \-E" , or .BR "grep \-F" . +. .SH "SEE ALSO" .BR grep (1), .BR xz (1), @@ -110,4 +140,5 @@ or .BR bzip2 (1), .BR lzop (1), .BR zstd (1), +.BR lz4 (1), .BR zgrep (1) diff --git a/src/scripts/xzgrep.in b/src/scripts/xzgrep.in index 490e47d..c9cd02d 100644 --- a/src/scripts/xzgrep.in +++ b/src/scripts/xzgrep.in @@ -1,4 +1,5 @@ #!@POSIX_SHELL@ +# SPDX-License-Identifier: GPL-2.0-or-later # xzgrep -- a wrapper around a grep program that decompresses files as needed # Adapted from a version sent by Charles Levert <charles@comm.polymtl.ca> @@ -34,7 +35,7 @@ case ${0##*/} in *) prog=xzgrep; grep=${GREP:-grep};; esac -version="$prog (@PACKAGE_NAME@) @VERSION@" +version="$prog (@PACKAGE_NAME@) @PACKAGE_VERSION@" usage="Usage: ${0##*/} [OPTION]... [-e] PATTERN [FILE]... Look for instances of PATTERN in the input FILEs, using their @@ -156,7 +157,7 @@ done eval "set -- $operands "'${1+"$@"}' if test $have_pat -eq 0; then - case ${1?"Missing pattern; try \`${0##*/} --help' for help"} in + case ${1?"Missing pattern; try '${0##*/} --help' for help"} in (*\'*) grep="$grep -e '"$(printf '%sX\n' "$1" | LC_ALL=C sed "$escape");; (*) @@ -180,6 +181,7 @@ for i; do *[-.]bz2 | *[-.]tbz | *.tbz2) uncompress="bzip2 -cdf";; *[-.]lzo | *[-.]tzo) uncompress="lzop -cdf";; *[-.]zst | *[-.]tzst) uncompress="zstd -cdfq";; # zstd needs -q. + *[-.]lz4) uncompress="lz4 -cdf";; *) uncompress="$xz -cdfqQ";; # -qQ to ignore warnings like unsupp. check. esac # xz_status will hold the decompressor's exit status. @@ -275,7 +277,7 @@ for i; do test "$(kill -l "$xz_status" 2> /dev/null)" != "PIPE" && exit "$xz_status" elif test "$xz_status" -gt 0; then # Decompression failed but we will continue with the remaining - # files anwyway. Set exit status to at least 2 to indicate an error. + # files anyway. Set exit status to at least 2 to indicate an error. test "$r" -lt 2 && r=2 fi diff --git a/src/scripts/xzless.1 b/src/scripts/xzless.1 index 2d05459..570e601 100644 --- a/src/scripts/xzless.1 +++ b/src/scripts/xzless.1 @@ -1,13 +1,11 @@ +.\" SPDX-License-Identifier: 0BSD .\" .\" Authors: Andrew Dudman .\" Lasse Collin .\" -.\" This file has been put into the public domain. -.\" You can do whatever you want with this file. -.\" .\" (Note that this file is not based on gzip's zless.1.) .\" -.TH XZLESS 1 "2010-09-27" "Tukaani" "XZ Utils" +.TH XZLESS 1 "2024-02-12" "Tukaani" "XZ Utils" .SH NAME xzless, lzless \- view xz or lzma compressed (text) files .SH SYNOPSIS @@ -19,10 +17,10 @@ xzless, lzless \- view xz or lzma compressed (text) files .SH DESCRIPTION .B xzless is a filter that displays text from compressed files to a terminal. -It works on files compressed with +Files supported by .BR xz (1) -or -.BR lzma (1). +are decompressed; +other files are assumed to be in uncompressed form already. If no .I files are given, diff --git a/src/scripts/xzless.in b/src/scripts/xzless.in index 79a849e..6346966 100644 --- a/src/scripts/xzless.in +++ b/src/scripts/xzless.in @@ -1,4 +1,5 @@ #!@POSIX_SHELL@ +# SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 1998, 2002, 2006, 2007 Free Software Foundation @@ -23,7 +24,7 @@ # specified via XZ_OPT. xz='@xz@ --format=auto' -version='xzless (@PACKAGE_NAME@) @VERSION@' +version='xzless (@PACKAGE_NAME@) @PACKAGE_VERSION@' usage="Usage: ${0##*/} [OPTION]... [FILE]... Like 'less', but operate on the uncompressed contents of xz compressed FILEs. @@ -47,13 +48,27 @@ if test "${LESSMETACHARS+set}" != set; then LESSMETACHARS="$space$tab$nl'"';*?"()<>[|&^`#\$%=~' fi -if test "$(less -V | { read _ ver _ && echo ${ver%%.*}; })" -ge 429; then +VER=$(less -V | { read _ ver _ && echo ${ver%%.*}; }) +if test "$VER" -ge 451; then + # less 451 or later: If the compressed file is valid but has + # zero bytes of uncompressed data, using two vertical bars ||- makes + # "less" check the exit status of xz and if it is zero then display + # an empty file. With a single vertical bar |- and no output from xz, + # "less" would attempt to display the raw input file instead. + LESSOPEN="||-$xz -cdfqQ -- %s" +elif test "$VER" -ge 429; then # less 429 or later: LESSOPEN pipe will be used on # standard input if $LESSOPEN begins with |-. LESSOPEN="|-$xz -cdfqQ -- %s" else LESSOPEN="|$xz -cdfqQ -- %s" fi + +SHOW_PREPROC_ERRORS= +if test "$VER" -ge 632; then + SHOW_PREPROC_ERRORS=--show-preproc-errors +fi + export LESSMETACHARS LESSOPEN -exec less "$@" +exec less $SHOW_PREPROC_ERRORS "$@" diff --git a/src/scripts/xzmore.1 b/src/scripts/xzmore.1 index 9613974..afe89b8 100644 --- a/src/scripts/xzmore.1 +++ b/src/scripts/xzmore.1 @@ -1,53 +1,63 @@ +.\" SPDX-License-Identifier: 0BSD .\" -.\" Original zdiff.1 for gzip: Jean-loup Gailly -.\" Modifications for XZ Utils: Lasse Collin +.\" Authors: Andrew Dudman +.\" Lasse Collin .\" -.\" License: GNU GPLv2+ +.\" (Note that this file is based on xzless.1 instead of gzip's zmore.1.) .\" -.TH XZMORE 1 "2013-06-30" "Tukaani" "XZ Utils" +.TH XZMORE 1 "2024-02-12" "Tukaani" "XZ Utils" .SH NAME xzmore, lzmore \- view xz or lzma compressed (text) files +. .SH SYNOPSIS .B xzmore -.RI [ file... ] +.RI [ file ...] .br .B lzmore -.RI [ file... ] +.RI [ file ...] +. .SH DESCRIPTION -.B xzmore -is a filter which allows examination of +.B xzmore +displays text from compressed files to a terminal using +.BR more (1). +Files supported by .BR xz (1) -or -.BR lzma (1) -compressed text files one screenful at a time -on a soft-copy terminal. +are decompressed; +other files are assumed to be in uncompressed form already. +If no +.I files +are given, +.B xzmore +reads from standard input. +See the +.BR more (1) +manual for the keyboard commands. .PP -To use a pager other than the default -.B more, -set environment variable -.B PAGER -to the name of the desired program. -The name +Note that scrolling backwards might not be possible +depending on the implementation of +.BR more (1). +This is because +.B xzmore +uses a pipe to pass the decompressed data to +.BR more (1). +.BR xzless (1) +uses +.BR less (1) +which provides more advanced features. +.PP +The command .B lzmore is provided for backward compatibility with LZMA Utils. +. +.SH ENVIRONMENT .TP -.BR e " or " q -When the prompt \-\-More\-\-(Next file: -.IR file ) -is printed, this command causes -.B xzmore -to exit. -.TP -.B s -When the prompt \-\-More\-\-(Next file: -.IR file ) -is printed, this command causes -.B xzmore -to skip the next file and continue. -.PP -For list of keyboard commands supported while actually viewing the -content of a file, refer to manual of the pager you use, usually +.B PAGER +If +.B PAGER +is set, +its value is used as the pager instead of .BR more (1). +. .SH "SEE ALSO" .BR more (1), .BR xz (1), diff --git a/src/scripts/xzmore.in b/src/scripts/xzmore.in index 5188fda..c571913 100644 --- a/src/scripts/xzmore.in +++ b/src/scripts/xzmore.in @@ -1,4 +1,5 @@ #!@POSIX_SHELL@ +# SPDX-License-Identifier: GPL-2.0-or-later # Copyright (C) 2001, 2002, 2007 Free Software Foundation # Copyright (C) 1992, 1993 Jean-loup Gailly @@ -23,7 +24,7 @@ # specified via XZ_OPT. xz='@xz@ --format=auto' -version='xzmore (@PACKAGE_NAME@) @VERSION@' +version='xzmore (@PACKAGE_NAME@) @PACKAGE_VERSION@' usage="Usage: ${0##*/} [OPTION]... [FILE]... Like 'more', but operate on the uncompressed contents of xz compressed FILEs. diff --git a/src/xz/Makefile.am b/src/xz/Makefile.am index 4bc64f3..4ff061f 100644 --- a/src/xz/Makefile.am +++ b/src/xz/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## bin_PROGRAMS = xz @@ -25,6 +21,8 @@ xz_SOURCES = \ options.c \ options.h \ private.h \ + sandbox.c \ + sandbox.h \ signals.c \ signals.h \ suffix.c \ @@ -53,7 +51,7 @@ xz_CPPFLAGS = \ -I$(top_srcdir)/src/liblzma/api \ -I$(top_builddir)/lib -xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(CAPSICUM_LIB) +xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la if COND_GNULIB xz_LDADD += $(top_builddir)/lib/libgnu.a diff --git a/src/xz/Makefile.in b/src/xz/Makefile.in index e6422c5..18e0151 100644 --- a/src/xz/Makefile.in +++ b/src/xz/Makefile.in @@ -98,8 +98,8 @@ bin_PROGRAMS = xz$(EXEEXT) @COND_LZMALINKS_TRUE@am__append_4 = lzma unlzma lzcat subdir = src/xz ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -127,23 +127,25 @@ PROGRAMS = $(bin_PROGRAMS) am__xz_SOURCES_DIST = args.c args.h coder.c coder.h file_io.c \ file_io.h hardware.c hardware.h main.c main.h message.c \ message.h mytime.c mytime.h options.c options.h private.h \ - signals.c signals.h suffix.c suffix.h util.c util.h \ - ../common/tuklib_open_stdxxx.c ../common/tuklib_progname.c \ - ../common/tuklib_exit.c ../common/tuklib_mbstr_width.c \ - ../common/tuklib_mbstr_fw.c list.c list.h xz_w32res.rc + sandbox.c sandbox.h signals.c signals.h suffix.c suffix.h \ + util.c util.h ../common/tuklib_open_stdxxx.c \ + ../common/tuklib_progname.c ../common/tuklib_exit.c \ + ../common/tuklib_mbstr_width.c ../common/tuklib_mbstr_fw.c \ + list.c list.h xz_w32res.rc @COND_MAIN_DECODER_TRUE@am__objects_1 = xz-list.$(OBJEXT) @COND_W32_TRUE@am__objects_2 = xz_w32res.$(OBJEXT) am_xz_OBJECTS = xz-args.$(OBJEXT) xz-coder.$(OBJEXT) \ xz-file_io.$(OBJEXT) xz-hardware.$(OBJEXT) xz-main.$(OBJEXT) \ xz-message.$(OBJEXT) xz-mytime.$(OBJEXT) xz-options.$(OBJEXT) \ - xz-signals.$(OBJEXT) xz-suffix.$(OBJEXT) xz-util.$(OBJEXT) \ - xz-tuklib_open_stdxxx.$(OBJEXT) xz-tuklib_progname.$(OBJEXT) \ - xz-tuklib_exit.$(OBJEXT) xz-tuklib_mbstr_width.$(OBJEXT) \ - xz-tuklib_mbstr_fw.$(OBJEXT) $(am__objects_1) $(am__objects_2) + xz-sandbox.$(OBJEXT) xz-signals.$(OBJEXT) xz-suffix.$(OBJEXT) \ + xz-util.$(OBJEXT) xz-tuklib_open_stdxxx.$(OBJEXT) \ + xz-tuklib_progname.$(OBJEXT) xz-tuklib_exit.$(OBJEXT) \ + xz-tuklib_mbstr_width.$(OBJEXT) xz-tuklib_mbstr_fw.$(OBJEXT) \ + $(am__objects_1) $(am__objects_2) xz_OBJECTS = $(am_xz_OBJECTS) am__DEPENDENCIES_1 = xz_DEPENDENCIES = $(top_builddir)/src/liblzma/liblzma.la \ - $(am__DEPENDENCIES_1) $(am__append_3) $(am__DEPENDENCIES_1) + $(am__append_3) $(am__DEPENDENCIES_1) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) am__v_lt_0 = --silent @@ -167,8 +169,9 @@ am__depfiles_remade = ./$(DEPDIR)/xz-args.Po ./$(DEPDIR)/xz-coder.Po \ ./$(DEPDIR)/xz-file_io.Po ./$(DEPDIR)/xz-hardware.Po \ ./$(DEPDIR)/xz-list.Po ./$(DEPDIR)/xz-main.Po \ ./$(DEPDIR)/xz-message.Po ./$(DEPDIR)/xz-mytime.Po \ - ./$(DEPDIR)/xz-options.Po ./$(DEPDIR)/xz-signals.Po \ - ./$(DEPDIR)/xz-suffix.Po ./$(DEPDIR)/xz-tuklib_exit.Po \ + ./$(DEPDIR)/xz-options.Po ./$(DEPDIR)/xz-sandbox.Po \ + ./$(DEPDIR)/xz-signals.Po ./$(DEPDIR)/xz-suffix.Po \ + ./$(DEPDIR)/xz-tuklib_exit.Po \ ./$(DEPDIR)/xz-tuklib_mbstr_fw.Po \ ./$(DEPDIR)/xz-tuklib_mbstr_width.Po \ ./$(DEPDIR)/xz-tuklib_open_stdxxx.Po \ @@ -259,7 +262,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -388,6 +390,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ @@ -408,8 +412,8 @@ top_srcdir = @top_srcdir@ xz = @xz@ xz_SOURCES = args.c args.h coder.c coder.h file_io.c file_io.h \ hardware.c hardware.h main.c main.h message.c message.h \ - mytime.c mytime.h options.c options.h private.h signals.c \ - signals.h suffix.c suffix.h util.c util.h \ + mytime.c mytime.h options.c options.h private.h sandbox.c \ + sandbox.h signals.c signals.h suffix.c suffix.h util.c util.h \ ../common/tuklib_open_stdxxx.c ../common/tuklib_progname.c \ ../common/tuklib_exit.c ../common/tuklib_mbstr_width.c \ ../common/tuklib_mbstr_fw.c $(am__append_1) $(am__append_2) @@ -421,8 +425,8 @@ xz_CPPFLAGS = \ # libgnu.a may need these libs, so this must be after libgnu.a. -xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(CAPSICUM_LIB) \ - $(am__append_3) $(LTLIBINTL) +xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(am__append_3) \ + $(LTLIBINTL) dist_man_MANS = xz.1 xzlinks = unxz xzcat $(am__append_4) all: all-am @@ -527,6 +531,7 @@ distclean-compile: @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-message.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-mytime.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-options.Po@am__quote@ # am--include-marker +@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-sandbox.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-signals.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-suffix.Po@am__quote@ # am--include-marker @AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_exit.Po@am__quote@ # am--include-marker @@ -675,6 +680,20 @@ xz-options.obj: options.c @AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ @am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-options.obj `if test -f 'options.c'; then $(CYGPATH_W) 'options.c'; else $(CYGPATH_W) '$(srcdir)/options.c'; fi` +xz-sandbox.o: sandbox.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-sandbox.o -MD -MP -MF $(DEPDIR)/xz-sandbox.Tpo -c -o xz-sandbox.o `test -f 'sandbox.c' || echo '$(srcdir)/'`sandbox.c +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-sandbox.Tpo $(DEPDIR)/xz-sandbox.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='sandbox.c' object='xz-sandbox.o' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-sandbox.o `test -f 'sandbox.c' || echo '$(srcdir)/'`sandbox.c + +xz-sandbox.obj: sandbox.c +@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-sandbox.obj -MD -MP -MF $(DEPDIR)/xz-sandbox.Tpo -c -o xz-sandbox.obj `if test -f 'sandbox.c'; then $(CYGPATH_W) 'sandbox.c'; else $(CYGPATH_W) '$(srcdir)/sandbox.c'; fi` +@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-sandbox.Tpo $(DEPDIR)/xz-sandbox.Po +@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='sandbox.c' object='xz-sandbox.obj' libtool=no @AMDEPBACKSLASH@ +@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@ +@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-sandbox.obj `if test -f 'sandbox.c'; then $(CYGPATH_W) 'sandbox.c'; else $(CYGPATH_W) '$(srcdir)/sandbox.c'; fi` + xz-signals.o: signals.c @am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-signals.o -MD -MP -MF $(DEPDIR)/xz-signals.Tpo -c -o xz-signals.o `test -f 'signals.c' || echo '$(srcdir)/'`signals.c @am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-signals.Tpo $(DEPDIR)/xz-signals.Po @@ -985,6 +1004,7 @@ distclean: distclean-am -rm -f ./$(DEPDIR)/xz-message.Po -rm -f ./$(DEPDIR)/xz-mytime.Po -rm -f ./$(DEPDIR)/xz-options.Po + -rm -f ./$(DEPDIR)/xz-sandbox.Po -rm -f ./$(DEPDIR)/xz-signals.Po -rm -f ./$(DEPDIR)/xz-suffix.Po -rm -f ./$(DEPDIR)/xz-tuklib_exit.Po @@ -1049,6 +1069,7 @@ maintainer-clean: maintainer-clean-am -rm -f ./$(DEPDIR)/xz-message.Po -rm -f ./$(DEPDIR)/xz-mytime.Po -rm -f ./$(DEPDIR)/xz-options.Po + -rm -f ./$(DEPDIR)/xz-sandbox.Po -rm -f ./$(DEPDIR)/xz-signals.Po -rm -f ./$(DEPDIR)/xz-suffix.Po -rm -f ./$(DEPDIR)/xz-tuklib_exit.Po diff --git a/src/xz/args.c b/src/xz/args.c index 17e778c..eba1b97 100644 --- a/src/xz/args.c +++ b/src/xz/args.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file args.c @@ -5,10 +7,8 @@ /// /// \note Filter-specific options parsing is in options.c. // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -83,14 +83,20 @@ parse_block_list(const char *str_const) ++count; // Prevent an unlikely integer overflow. - if (count > SIZE_MAX / sizeof(uint64_t) - 1) + if (count > SIZE_MAX / sizeof(block_list_entry) - 1) message_fatal(_("%s: Too many arguments to --block-list"), str); // Allocate memory to hold all the sizes specified. // If --block-list was specified already, its value is forgotten. free(opt_block_list); - opt_block_list = xmalloc((count + 1) * sizeof(uint64_t)); + opt_block_list = xmalloc((count + 1) * sizeof(block_list_entry)); + + // Clear the bitmask of filter chains in use. + block_list_chain_mask = 0; + + // Reset the largest Block size found in --block-list. + block_list_largest = 0; for (size_t i = 0; i < count; ++i) { // Locate the next comma and replace it with \0. @@ -98,6 +104,43 @@ parse_block_list(const char *str_const) if (p != NULL) *p = '\0'; + // Use the default filter chain unless overridden. + opt_block_list[i].chain_num = 0; + + // To specify a filter chain, the block list entry may be + // prepended with "[filter-chain-number]:". The size is + // still required for every block. + // For instance: + // --block-list=2:10MiB,1:5MiB,,8MiB,0:0 + // + // Translates to: + // 1. Block of 10 MiB using filter chain 2 + // 2. Block of 5 MiB using filter chain 1 + // 3. Block of 5 MiB using filter chain 1 + // 4. Block of 8 MiB using the default filter chain + // 5. The last block uses the default filter chain + // + // The block list: + // --block-list=2:MiB,1:,0 + // + // Is not allowed because the second block does not specify + // the block size, only the filter chain. + if (str[0] >= '0' && str[0] <= '9' && str[1] == ':') { + if (str[2] == '\0') + message_fatal(_("In --block-list, block " + "size is missing after " + "filter chain number '%c:'"), + str[0]); + + const unsigned chain_num = (unsigned)(str[0] - '0'); + opt_block_list[i].chain_num = chain_num; + block_list_chain_mask |= 1U << chain_num; + str += 2; + } else { + // This Block uses the default filter chain. + block_list_chain_mask |= 1U << 0; + } + if (str[0] == '\0') { // There is no string, that is, a comma follows // another comma. Use the previous value. @@ -107,25 +150,39 @@ parse_block_list(const char *str_const) assert(i > 0); opt_block_list[i] = opt_block_list[i - 1]; } else { - opt_block_list[i] = str_to_uint64("block-list", str, - 0, UINT64_MAX); + opt_block_list[i].size = str_to_uint64("block-list", + str, 0, UINT64_MAX); // Zero indicates no more new Blocks. - if (opt_block_list[i] == 0) { + if (opt_block_list[i].size == 0) { if (i + 1 != count) message_fatal(_("0 can only be used " "as the last element " "in --block-list")); - opt_block_list[i] = UINT64_MAX; + opt_block_list[i].size = UINT64_MAX; } + + // Remember the largest Block size in the list. + // + // NOTE: Do this after handling the special value 0 + // because when 0 is used, we don't want to reduce + // the Block size of the multithreaded encoder. + if (block_list_largest < opt_block_list[i].size) + block_list_largest = opt_block_list[i].size; } - str = p + 1; + // Be standards compliant: p + 1 is undefined behavior + // if p == NULL. That occurs on the last iteration of + // the loop when we won't care about the value of str + // anymore anyway. That is, this is done conditionally + // solely for standard conformance reasons. + if (p != NULL) + str = p + 1; } // Terminate the array. - opt_block_list[count] = 0; + opt_block_list[count].size = 0; free(str_start); return; @@ -136,13 +193,26 @@ static void parse_real(args_info *args, int argc, char **argv) { enum { - OPT_X86 = INT_MIN, + OPT_FILTERS = INT_MIN, + OPT_FILTERS1, + OPT_FILTERS2, + OPT_FILTERS3, + OPT_FILTERS4, + OPT_FILTERS5, + OPT_FILTERS6, + OPT_FILTERS7, + OPT_FILTERS8, + OPT_FILTERS9, + OPT_FILTERS_HELP, + + OPT_X86, OPT_POWERPC, OPT_IA64, OPT_ARM, OPT_ARMTHUMB, OPT_ARM64, OPT_SPARC, + OPT_RISCV, OPT_DELTA, OPT_LZMA1, OPT_LZMA2, @@ -191,7 +261,7 @@ parse_real(args_info *args, int argc, char **argv) { "check", required_argument, NULL, 'C' }, { "ignore-check", no_argument, NULL, OPT_IGNORE_CHECK }, { "block-size", required_argument, NULL, OPT_BLOCK_SIZE }, - { "block-list", required_argument, NULL, OPT_BLOCK_LIST }, + { "block-list", required_argument, NULL, OPT_BLOCK_LIST }, { "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS }, { "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS }, { "memlimit-mt-decompress", required_argument, NULL, OPT_MEM_MT_DECOMPRESS }, @@ -206,6 +276,18 @@ parse_real(args_info *args, int argc, char **argv) { "best", no_argument, NULL, '9' }, // Filters + { "filters", optional_argument, NULL, OPT_FILTERS}, + { "filters1", optional_argument, NULL, OPT_FILTERS1}, + { "filters2", optional_argument, NULL, OPT_FILTERS2}, + { "filters3", optional_argument, NULL, OPT_FILTERS3}, + { "filters4", optional_argument, NULL, OPT_FILTERS4}, + { "filters5", optional_argument, NULL, OPT_FILTERS5}, + { "filters6", optional_argument, NULL, OPT_FILTERS6}, + { "filters7", optional_argument, NULL, OPT_FILTERS7}, + { "filters8", optional_argument, NULL, OPT_FILTERS8}, + { "filters9", optional_argument, NULL, OPT_FILTERS9}, + { "filters-help", optional_argument, NULL, OPT_FILTERS_HELP}, + { "lzma1", optional_argument, NULL, OPT_LZMA1 }, { "lzma2", optional_argument, NULL, OPT_LZMA2 }, { "x86", optional_argument, NULL, OPT_X86 }, @@ -215,6 +297,7 @@ parse_real(args_info *args, int argc, char **argv) { "armthumb", optional_argument, NULL, OPT_ARMTHUMB }, { "arm64", optional_argument, NULL, OPT_ARM64 }, { "sparc", optional_argument, NULL, OPT_SPARC }, + { "riscv", optional_argument, NULL, OPT_RISCV }, { "delta", optional_argument, NULL, OPT_DELTA }, // Other options @@ -372,7 +455,30 @@ parse_real(args_info *args, int argc, char **argv) opt_mode = MODE_COMPRESS; break; - // Filter setup + // --filters + case OPT_FILTERS: + coder_add_filters_from_str(optarg); + break; + + // --filters1...--filters9 + case OPT_FILTERS1: + case OPT_FILTERS2: + case OPT_FILTERS3: + case OPT_FILTERS4: + case OPT_FILTERS5: + case OPT_FILTERS6: + case OPT_FILTERS7: + case OPT_FILTERS8: + case OPT_FILTERS9: + coder_add_block_filters(optarg, + (size_t)(c - OPT_FILTERS)); + break; + + // --filters-help + case OPT_FILTERS_HELP: + // This doesn't return. + message_filters_help(); + break; case OPT_X86: coder_add_filter(LZMA_FILTER_X86, @@ -409,6 +515,11 @@ parse_real(args_info *args, int argc, char **argv) options_bcj(optarg)); break; + case OPT_RISCV: + coder_add_filter(LZMA_FILTER_RISCV, + options_bcj(optarg)); + break; + case OPT_DELTA: coder_add_filter(LZMA_FILTER_DELTA, options_delta(optarg)); @@ -516,8 +627,8 @@ parse_real(args_info *args, int argc, char **argv) case OPT_FILES0: if (args->files_name != NULL) message_fatal(_("Only one file can be " - "specified with `--files' " - "or `--files0'.")); + "specified with '--files' " + "or '--files0'.")); if (optarg == NULL) { args->files_name = stdin_filename; @@ -718,6 +829,39 @@ args_parse(args_info *args, int argc, char **argv) if (opt_mode == MODE_COMPRESS && opt_format == FORMAT_AUTO) opt_format = FORMAT_XZ; + // Set opt_block_list to NULL if we are not compressing to the .xz + // format. This option cannot be used outside of this case, and + // simplifies the implementation later. + if ((opt_mode != MODE_COMPRESS || opt_format != FORMAT_XZ) + && opt_block_list != NULL) { + message(V_WARNING, _("--block-list is ignored unless " + "compressing to the .xz format")); + free(opt_block_list); + opt_block_list = NULL; + } + + // If raw format is used and a custom suffix is not provided, + // then only stdout mode can be used when compressing or + // decompressing. + if (opt_format == FORMAT_RAW && !suffix_is_set() && !opt_stdout + && (opt_mode == MODE_COMPRESS + || opt_mode == MODE_DECOMPRESS)) { + if (args->files_name != NULL) + message_fatal(_("With --format=raw, " + "--suffix=.SUF is required " + "unless writing to stdout")); + + // If all of the filenames provided are "-" (more than one + // "-" could be specified) or no filenames are provided, + // then we are only going to be writing to standard out. + for (int i = optind; i < argc; i++) { + if (strcmp(argv[i], "-") != 0) + message_fatal(_("With --format=raw, " + "--suffix=.SUF is required " + "unless writing to stdout")); + } + } + // Compression settings need to be validated (options themselves and // their memory usage) when compressing to any file format. It has to // be done also when uncompressing raw data, since for raw decoding @@ -727,14 +871,6 @@ args_parse(args_info *args, int argc, char **argv) && opt_mode != MODE_LIST)) coder_set_compression_settings(); - // If raw format is used and a custom suffix is not provided, - // then only stdout mode can be used when compressing or decompressing. - if (opt_format == FORMAT_RAW && !suffix_is_set() && !opt_stdout - && (opt_mode == MODE_COMPRESS - || opt_mode == MODE_DECOMPRESS)) - message_fatal(_("With --format=raw, --suffix=.SUF is " - "required unless writing to stdout")); - // If no filenames are given, use stdin. if (argv[optind] == NULL && args->files_name == NULL) { // We don't modify or free() the "-" constant. The caller diff --git a/src/xz/args.h b/src/xz/args.h index a1a5930..e693ecd 100644 --- a/src/xz/args.h +++ b/src/xz/args.h @@ -1,12 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file args.h /// \brief Argument parsing // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// diff --git a/src/xz/coder.c b/src/xz/coder.c index 91d40ed..5e41f0d 100644 --- a/src/xz/coder.c +++ b/src/xz/coder.c @@ -1,16 +1,17 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file coder.c /// \brief Compresses or uncompresses a file // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// #include "private.h" +#include "tuklib_integer.h" /// Return value type for coder_init(). @@ -26,25 +27,47 @@ enum format_type opt_format = FORMAT_AUTO; bool opt_auto_adjust = true; bool opt_single_stream = false; uint64_t opt_block_size = 0; -uint64_t *opt_block_list = NULL; - +block_list_entry *opt_block_list = NULL; +uint64_t block_list_largest; +uint32_t block_list_chain_mask; /// Stream used to communicate with liblzma static lzma_stream strm = LZMA_STREAM_INIT; -/// Filters needed for all encoding all formats, and also decoding in raw data -static lzma_filter filters[LZMA_FILTERS_MAX + 1]; +/// Maximum number of filter chains. The first filter chain is the default, +/// and 9 other filter chains can be specified with --filtersX. +#define NUM_FILTER_CHAIN_MAX 10 + +/// The default filter chain is in chains[0]. It is used for encoding +/// in all supported formats and also for decdoing raw streams. The other +/// filter chains are set by --filtersX to support changing filters with +/// the --block-list option. +static lzma_filter chains[NUM_FILTER_CHAIN_MAX][LZMA_FILTERS_MAX + 1]; + +/// Bitmask indicating which filter chains are actually used when encoding +/// in the .xz format. This is needed since the filter chains specified using +/// --filtersX (or the default filter chain) might in reality be unneeded +/// if they are never used in --block-list. When --block-list isn't +/// specified, only the default filter chain is used, thus the initial +/// value of this variable is 1U << 0 (the number of the default chain is 0). +static uint32_t chains_used_mask = 1U << 0; /// Input and output buffers static io_buf in_buf; static io_buf out_buf; -/// Number of filters. Zero indicates that we are using a preset. +/// Number of filters in the default filter chain. Zero indicates that +/// we are using a preset. static uint32_t filters_count = 0; /// Number of the preset (0-9) static uint32_t preset_number = LZMA_PRESET_DEFAULT; +/// True if the current default filter chain was set using the --filters +/// option. The filter chain is reset if a preset option (like -9) or an +/// old-style filter option (like --lzma2) is used after a --filters option. +static bool string_to_filter_used = false; + /// Integrity check type static lzma_check check; @@ -60,7 +83,6 @@ static bool allow_trailing_input; static lzma_mt mt_options = { .flags = 0, .timeout = 300, - .filters = filters, }; #endif @@ -77,14 +99,14 @@ coder_set_check(lzma_check new_check) static void forget_filter_chain(void) { - // Setting a preset makes us forget a possibly defined custom - // filter chain. - while (filters_count > 0) { - --filters_count; - free(filters[filters_count].options); - filters[filters_count].options = NULL; + // Setting a preset or using --filters makes us forget + // the earlier custom filter chain (if any). + if (filters_count > 0) { + lzma_filters_free(chains[0], NULL); + filters_count = 0; } + string_to_filter_used = false; return; } @@ -114,9 +136,15 @@ coder_add_filter(lzma_vli id, void *options) if (filters_count == LZMA_FILTERS_MAX) message_fatal(_("Maximum number of filters is four")); - filters[filters_count].id = id; - filters[filters_count].options = options; - ++filters_count; + if (string_to_filter_used) + forget_filter_chain(); + + chains[0][filters_count].id = id; + chains[0][filters_count].options = options; + + // Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify + // implementation of forget_filter_chain(). + chains[0][++filters_count].id = LZMA_VLI_UNKNOWN; // Setting a custom filter chain makes us forget the preset options. // This makes a difference if one specifies e.g. "xz -9 --lzma2 -e" @@ -128,6 +156,69 @@ coder_add_filter(lzma_vli id, void *options) } +static void +str_to_filters(const char *str, uint32_t index, uint32_t flags) +{ + int error_pos; + const char *err = lzma_str_to_filters(str, &error_pos, + chains[index], flags, NULL); + + if (err != NULL) { + char filter_num[2] = ""; + if (index > 0) + filter_num[0] = '0' + index; + + // FIXME? The message in err isn't translated. + // Including the translations in the xz translations is + // slightly ugly but possible. Creating a new domain for + // liblzma might not be worth it especially since on some + // OSes it adds extra dependencies to translation libraries. + message(V_ERROR, _("Error in --filters%s=FILTERS option:"), + filter_num); + message(V_ERROR, "%s", str); + message(V_ERROR, "%*s^", error_pos, ""); + message_fatal("%s", err); + } +} + + +extern void +coder_add_filters_from_str(const char *filter_str) +{ + // Forget presets and previously defined filter chain. See + // coder_add_filter() above for why preset_number must be reset too. + forget_filter_chain(); + preset_number = LZMA_PRESET_DEFAULT; + + string_to_filter_used = true; + + // Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw. + str_to_filters(filter_str, 0, LZMA_STR_ALL_FILTERS); + + // Set the filters_count to be the number of filters converted from + // the string. + for (filters_count = 0; chains[0][filters_count].id + != LZMA_VLI_UNKNOWN; + ++filters_count) ; + + assert(filters_count > 0); + return; +} + + +extern void +coder_add_block_filters(const char *str, size_t slot) +{ + // Free old filters first, if they were previously allocated. + if (chains_used_mask & (1U << slot)) + lzma_filters_free(chains[slot], NULL); + + str_to_filters(str, slot, 0); + + chains_used_mask |= 1U << slot; +} + + tuklib_attr_noreturn static void memlimit_too_small(uint64_t memory_usage) @@ -139,6 +230,67 @@ memlimit_too_small(uint64_t memory_usage) } +#ifdef HAVE_ENCODERS +/// \brief Calculate the memory usage of each filter chain. +/// +/// \param chains_memusages If non-NULL, the memusage of the encoder +/// or decoder for each chain is stored in +/// this array. +/// \param mt If non-NULL, calculate memory usage of +/// multithreaded encoder. +/// \param encode Whether to calculate encoder or decoder +/// memory usage. This must be true if +/// mt != NULL. +/// +/// \return Return the highest memory usage of all of the filter chains. +static uint64_t +get_chains_memusage(uint64_t *chains_memusages, const lzma_mt *mt, bool encode) +{ + uint64_t max_memusage = 0; + +#ifdef MYTHREAD_ENABLED + // Copy multithreading options to a temporary struct since the + // "filters" member needs to be changed. + lzma_mt mt_local; + if (mt != NULL) + mt_local = *mt; +#else + (void)mt; +#endif + + for (uint32_t i = 0; i < ARRAY_SIZE(chains); i++) { + if (!(chains_used_mask & (1U << i))) + continue; + + uint64_t memusage = UINT64_MAX; +#ifdef MYTHREAD_ENABLED + if (mt != NULL) { + assert(encode); + mt_local.filters = chains[i]; + memusage = lzma_stream_encoder_mt_memusage(&mt_local); + } else +#endif + if (encode) { + memusage = lzma_raw_encoder_memusage(chains[i]); + } +#ifdef HAVE_DECODERS + else { + memusage = lzma_raw_decoder_memusage(chains[i]); + } +#endif + + if (chains_memusages != NULL) + chains_memusages[i] = memusage; + + if (memusage > max_memusage) + max_memusage = memusage; + } + + return max_memusage; +} +#endif + + extern void coder_set_compression_settings(void) { @@ -156,10 +308,56 @@ coder_set_compression_settings(void) check = LZMA_CHECK_CRC32; } +#ifdef HAVE_ENCODERS + if (opt_block_list != NULL) { + // args.c ensures these. + assert(opt_mode == MODE_COMPRESS); + assert(opt_format == FORMAT_XZ); + + // Find out if block_list_chain_mask has a bit set that + // isn't set in chains_used_mask. + const uint32_t missing_chains_mask + = (block_list_chain_mask ^ chains_used_mask) + & block_list_chain_mask; + + // If a filter chain was specified in --block-list but no + // matching --filtersX option was used, exit with an error. + if (missing_chains_mask != 0) { + // Get the number of the first missing filter chain + // and show it in the error message. + const unsigned first_missing + = (unsigned)ctz32(missing_chains_mask); + + message_fatal(_("filter chain %u used by " + "--block-list but not specified " + "with --filters%u="), + first_missing, first_missing); + } + + // Omit the unused filter chains from mask of used chains. + // + // (FIXME? When built with debugging, coder_free() will free() + // the filter chains (except the default chain) which makes + // Valgrind show fewer reachable allocations. But coder_free() + // uses this mask to determine which chains to free. Thus it + // won't free the ones that are cleared here from the mask. + // In practice this doesn't matter.) + chains_used_mask &= block_list_chain_mask; + } else { + // Reset filters used mask in case --block-list is not + // used, but --filtersX is used. + chains_used_mask = 1U << 0; + } +#endif + // Options for LZMA1 or LZMA2 in case we are using a preset. static lzma_options_lzma opt_lzma; - if (filters_count == 0) { + // The first filter in the chains[] array is for the default + // filter chain. + lzma_filter *default_filters = chains[0]; + + if (filters_count == 0 && chains_used_mask & 1) { // We are using a preset. This is not a good idea in raw mode // except when playing around with things. Different versions // of this software may use different options in presets, and @@ -179,46 +377,61 @@ coder_set_compression_settings(void) message_bug(); // Use LZMA2 except with --format=lzma we use LZMA1. - filters[0].id = opt_format == FORMAT_LZMA + default_filters[0].id = opt_format == FORMAT_LZMA ? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2; - filters[0].options = &opt_lzma; + default_filters[0].options = &opt_lzma; + filters_count = 1; - } - // Terminate the filter options array. - filters[filters_count].id = LZMA_VLI_UNKNOWN; + // Terminate the filter options array. + default_filters[1].id = LZMA_VLI_UNKNOWN; + } // If we are using the .lzma format, allow exactly one filter - // which has to be LZMA1. + // which has to be LZMA1. There is no need to check if the default + // filter chain is being used since it can only be disabled if + // --block-list is used, which is incompatible with FORMAT_LZMA. if (opt_format == FORMAT_LZMA && (filters_count != 1 - || filters[0].id != LZMA_FILTER_LZMA1)) + || default_filters[0].id != LZMA_FILTER_LZMA1)) message_fatal(_("The .lzma format supports only " "the LZMA1 filter")); // If we are using the .xz format, make sure that there is no LZMA1 - // filter to prevent LZMA_PROG_ERROR. - if (opt_format == FORMAT_XZ) + // filter to prevent LZMA_PROG_ERROR. With the chains from --filtersX + // we have already ensured this by calling lzma_str_to_filters() + // without setting the flags that would allow non-.xz filters. + if (opt_format == FORMAT_XZ && chains_used_mask & 1) for (size_t i = 0; i < filters_count; ++i) - if (filters[i].id == LZMA_FILTER_LZMA1) + if (default_filters[i].id == LZMA_FILTER_LZMA1) message_fatal(_("LZMA1 cannot be used " "with the .xz format")); - // Print the selected filter chain. - message_filters_show(V_DEBUG, filters); + if (chains_used_mask & 1) { + // Print the selected default filter chain. + message_filters_show(V_DEBUG, default_filters); + } // The --flush-timeout option requires LZMA_SYNC_FLUSH support - // from the filter chain. Currently threaded encoder doesn't support - // LZMA_SYNC_FLUSH so single-threaded mode must be used. + // from the filter chain. Currently the threaded encoder doesn't + // support LZMA_SYNC_FLUSH so single-threaded mode must be used. if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) { - for (size_t i = 0; i < filters_count; ++i) { - switch (filters[i].id) { - case LZMA_FILTER_LZMA2: - case LZMA_FILTER_DELTA: - break; + for (unsigned i = 0; i < ARRAY_SIZE(chains); ++i) { + if (!(chains_used_mask & (1U << i))) + continue; + + const lzma_filter *fc = chains[i]; + for (size_t j = 0; fc[j].id != LZMA_VLI_UNKNOWN; j++) { + switch (fc[j].id) { + case LZMA_FILTER_LZMA2: + case LZMA_FILTER_DELTA: + break; - default: - message_fatal(_("The filter chain is " - "incompatible with --flush-timeout")); + default: + message_fatal(_("Filter chain %u is " + "incompatible with " + "--flush-timeout"), + i); + } } } @@ -229,23 +442,72 @@ coder_set_compression_settings(void) } } - // Get the memory usage. Note that if --format=raw was used, - // we can be decompressing. + // Get memory limit and the memory usage of the used filter chains. + // Note that if --format=raw was used, we can be decompressing + // using the default filter chain. // - // If multithreaded .xz compression is done, this value will be - // replaced. + // If multithreaded .xz compression is done, the memory limit + // will be replaced. uint64_t memory_limit = hardware_memlimit_get(opt_mode); uint64_t memory_usage = UINT64_MAX; + +#ifdef HAVE_ENCODERS + // Memory usage for each encoder filter chain (default + // or --filtersX). The encoder options may need to be + // scaled down depending on the memory usage limit. + uint64_t encoder_memusages[ARRAY_SIZE(chains)]; +#endif + if (opt_mode == MODE_COMPRESS) { #ifdef HAVE_ENCODERS # ifdef MYTHREAD_ENABLED if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) { memory_limit = hardware_memlimit_mtenc_get(); mt_options.threads = hardware_threads_get(); - mt_options.block_size = opt_block_size; + + uint64_t block_size = opt_block_size; + + // If opt_block_size is not set, find the maximum + // recommended Block size based on the filter chains + if (block_size == 0) { + for (unsigned i = 0; i < ARRAY_SIZE(chains); + i++) { + if (!(chains_used_mask & (1U << i))) + continue; + + uint64_t size = lzma_mt_block_size( + chains[i]); + + // If this returns an error, then one + // of the filter chains in use is + // invalid, so there is no point in + // progressing further. + if (size == UINT64_MAX) + message_fatal(_("Unsupported " + "options in filter " + "chain %u"), i); + + if (size > block_size) + block_size = size; + } + + // If --block-list was used and our current + // Block size exceeds the largest size + // in --block-list, reduce the Block size of + // the multithreaded encoder. The extra size + // would only be a waste of RAM. With a + // smaller Block size we might even be able + // to use more threads in some cases. + if (block_list_largest > 0 && block_size + > block_list_largest) + block_size = block_list_largest; + } + + mt_options.block_size = block_size; mt_options.check = check; - memory_usage = lzma_stream_encoder_mt_memusage( - &mt_options); + + memory_usage = get_chains_memusage(encoder_memusages, + &mt_options, true); if (memory_usage != UINT64_MAX) message(V_DEBUG, _("Using up to %" PRIu32 " threads."), @@ -253,12 +515,13 @@ coder_set_compression_settings(void) } else # endif { - memory_usage = lzma_raw_encoder_memusage(filters); + memory_usage = get_chains_memusage(encoder_memusages, + NULL, true); } #endif } else { #ifdef HAVE_DECODERS - memory_usage = lzma_raw_decoder_memusage(filters); + memory_usage = lzma_raw_decoder_memusage(default_filters); #endif } @@ -268,12 +531,13 @@ coder_set_compression_settings(void) // Print memory usage info before possible dictionary // size auto-adjusting. // - // NOTE: If only encoder support was built, we cannot show the + // NOTE: If only encoder support was built, we cannot show // what the decoder memory usage will be. message_mem_needed(V_DEBUG, memory_usage); -#ifdef HAVE_DECODERS - if (opt_mode == MODE_COMPRESS) { - const uint64_t decmem = lzma_raw_decoder_memusage(filters); + +#if defined(HAVE_ENCODERS) && defined(HAVE_DECODERS) + if (opt_mode == MODE_COMPRESS && message_verbosity_get() >= V_DEBUG) { + const uint64_t decmem = get_chains_memusage(NULL, NULL, false); if (decmem != UINT64_MAX) message(V_DEBUG, _("Decompression will need " "%s MiB of memory."), uint64_to_str( @@ -300,14 +564,21 @@ coder_set_compression_settings(void) // Reduce the number of threads by one and check // the memory usage. --mt_options.threads; - memory_usage = lzma_stream_encoder_mt_memusage( - &mt_options); + memory_usage = get_chains_memusage(encoder_memusages, + &mt_options, true); if (memory_usage == UINT64_MAX) message_bug(); if (memory_usage <= memory_limit) { // The memory usage is now low enough. - message(V_WARNING, _("Reduced the number of " + // + // Since 5.6.1: This is only shown at + // V_DEBUG instead of V_WARNING because + // changing the number of threads doesn't + // affect the output. On some systems this + // message would be too common now that + // multithreaded compression is the default. + message(V_DEBUG, _("Reduced the number of " "threads from %s to %s to not exceed " "the memory usage limit of %s MiB"), uint64_to_str( @@ -326,8 +597,11 @@ coder_set_compression_settings(void) // way -T0 won't use insane amount of memory but at the same // time the soft limit will never make xz fail and never make // xz change settings that would affect the compressed output. + // + // Since 5.6.1: Like above, this is now shown at V_DEBUG + // instead of V_WARNING. if (hardware_memlimit_mtenc_is_default()) { - message(V_WARNING, _("Reduced the number of threads " + message(V_DEBUG, _("Reduced the number of threads " "from %s to one. The automatic memory usage " "limit of %s MiB is still being exceeded. " "%s MiB of memory is required. " @@ -353,7 +627,8 @@ coder_set_compression_settings(void) // the multithreaded mode but the output // is also different. hardware_threads_set(1); - memory_usage = lzma_raw_encoder_memusage(filters); + memory_usage = get_chains_memusage(encoder_memusages, + NULL, true); message(V_WARNING, _("Switching to single-threaded mode " "to not exceed the memory usage limit of %s MiB"), uint64_to_str(round_up_to_mib(memory_limit), 0)); @@ -368,55 +643,85 @@ coder_set_compression_settings(void) if (!opt_auto_adjust) memlimit_too_small(memory_usage); - // Look for the last filter if it is LZMA2 or LZMA1, so we can make - // it use less RAM. With other filters we don't know what to do. - size_t i = 0; - while (filters[i].id != LZMA_FILTER_LZMA2 - && filters[i].id != LZMA_FILTER_LZMA1) { - if (filters[i].id == LZMA_VLI_UNKNOWN) - memlimit_too_small(memory_usage); + // Adjust each filter chain that is exceeding the memory usage limit. + for (unsigned i = 0; i < ARRAY_SIZE(chains); i++) { + // Skip unused chains. + if (!(chains_used_mask & (1U << i))) + continue; + + // Skip chains that already meet the memory usage limit. + if (encoder_memusages[i] <= memory_limit) + continue; + + // Look for the last filter if it is LZMA2 or LZMA1, so we + // can make it use less RAM. We cannot adjust other filters. + unsigned j = 0; + while (chains[i][j].id != LZMA_FILTER_LZMA2 + && chains[i][j].id != LZMA_FILTER_LZMA1) { + // NOTE: This displays the too high limit of this + // particular filter chain. If multiple chains are + // specified and another one would need more then + // this message could be confusing. As long as LZMA2 + // is the only memory hungry filter in .xz this + // doesn't matter at all in practice. + // + // FIXME? However, it's sort of odd still if we had + // switched from multithreaded mode to single-threaded + // mode because single-threaded produces different + // output. So the messages could perhaps be clearer. + // Another case of this is a few lines below. + if (chains[i][j].id == LZMA_VLI_UNKNOWN) + memlimit_too_small(encoder_memusages[i]); + + ++j; + } - ++i; - } + // Decrease the dictionary size until we meet the memory + // usage limit. First round down to full mebibytes. + lzma_options_lzma *opt = chains[i][j].options; + const uint32_t orig_dict_size = opt->dict_size; + opt->dict_size &= ~((UINT32_C(1) << 20) - 1); - // Decrease the dictionary size until we meet the memory - // usage limit. First round down to full mebibytes. - lzma_options_lzma *opt = filters[i].options; - const uint32_t orig_dict_size = opt->dict_size; - opt->dict_size &= ~((UINT32_C(1) << 20) - 1); - while (true) { - // If it is below 1 MiB, auto-adjusting failed. We could be - // more sophisticated and scale it down even more, but let's - // see if many complain about this version. - // - // FIXME: Displays the scaled memory usage instead - // of the original. - if (opt->dict_size < (UINT32_C(1) << 20)) - memlimit_too_small(memory_usage); + while (true) { + // If it is below 1 MiB, auto-adjusting failed. + // + // FIXME? See the FIXME a few lines above. + if (opt->dict_size < (UINT32_C(1) << 20)) + memlimit_too_small(encoder_memusages[i]); - memory_usage = lzma_raw_encoder_memusage(filters); - if (memory_usage == UINT64_MAX) - message_bug(); + encoder_memusages[i] + = lzma_raw_encoder_memusage(chains[i]); + if (encoder_memusages[i] == UINT64_MAX) + message_bug(); - // Accept it if it is low enough. - if (memory_usage <= memory_limit) - break; + // Accept it if it is low enough. + if (encoder_memusages[i] <= memory_limit) + break; - // Otherwise 1 MiB down and try again. I hope this - // isn't too slow method for cases where the original - // dict_size is very big. - opt->dict_size -= UINT32_C(1) << 20; - } + // Otherwise adjust it 1 MiB down and try again. + opt->dict_size -= UINT32_C(1) << 20; + } - // Tell the user that we decreased the dictionary size. - message(V_WARNING, _("Adjusted LZMA%c dictionary size " - "from %s MiB to %s MiB to not exceed " - "the memory usage limit of %s MiB"), - filters[i].id == LZMA_FILTER_LZMA2 - ? '2' : '1', - uint64_to_str(orig_dict_size >> 20, 0), - uint64_to_str(opt->dict_size >> 20, 1), - uint64_to_str(round_up_to_mib(memory_limit), 2)); + // Tell the user that we decreased the dictionary size. + // The message is slightly different between the default + // filter chain (0) or and chains from --filtersX. + const char lzma_num = chains[i][j].id == LZMA_FILTER_LZMA2 + ? '2' : '1'; + const char *from_size = uint64_to_str(orig_dict_size >> 20, 0); + const char *to_size = uint64_to_str(opt->dict_size >> 20, 1); + const char *limit_size = uint64_to_str(round_up_to_mib( + memory_limit), 2); + if (i == 0) + message(V_WARNING, _("Adjusted LZMA%c dictionary " + "size from %s MiB to %s MiB to not exceed the " + "memory usage limit of %s MiB"), + lzma_num, from_size, to_size, limit_size); + else + message(V_WARNING, _("Adjusted LZMA%c dictionary size " + "for --filters%u from %s MiB to %s MiB to not " + "exceed the memory usage limit of %s MiB"), + lzma_num, i, from_size, to_size, limit_size); + } #endif return; @@ -512,6 +817,13 @@ coder_init(file_pair *pair) // These will be handled later in this function. allow_trailing_input = false; + // Set the first filter chain. If the --block-list option is not + // used then use the default filter chain (chains[0]). + // Otherwise, use first filter chain from the block list. + lzma_filter *active_filters = opt_block_list == NULL + ? chains[0] + : chains[opt_block_list[0].chain_num]; + if (opt_mode == MODE_COMPRESS) { #ifdef HAVE_ENCODERS switch (opt_format) { @@ -522,17 +834,19 @@ coder_init(file_pair *pair) case FORMAT_XZ: # ifdef MYTHREAD_ENABLED + mt_options.filters = active_filters; if (hardware_threads_is_mt()) ret = lzma_stream_encoder_mt( &strm, &mt_options); else # endif ret = lzma_stream_encoder( - &strm, filters, check); + &strm, active_filters, check); break; case FORMAT_LZMA: - ret = lzma_alone_encoder(&strm, filters[0].options); + ret = lzma_alone_encoder(&strm, + active_filters[0].options); break; # ifdef HAVE_LZIP_DECODER @@ -544,7 +858,7 @@ coder_init(file_pair *pair) # endif case FORMAT_RAW: - ret = lzma_raw_encoder(&strm, filters); + ret = lzma_raw_encoder(&strm, active_filters); break; } #endif @@ -668,7 +982,7 @@ coder_init(file_pair *pair) case FORMAT_RAW: // Memory usage has already been checked in // coder_set_compression_settings(). - ret = lzma_raw_decoder(&strm, filters); + ret = lzma_raw_decoder(&strm, active_filters); break; } @@ -716,6 +1030,7 @@ coder_init(file_pair *pair) } +#ifdef HAVE_ENCODERS /// Resolve conflicts between opt_block_size and opt_block_list in single /// threaded mode. We want to default to opt_block_list, except when it is /// larger than opt_block_size. If this is the case for the current Block @@ -746,12 +1061,39 @@ split_block(uint64_t *block_remaining, } else { // The Block at *list_pos has been finished. Go to the next - // entry in the list. If the end of the list has been reached, - // reuse the size of the last Block. - if (opt_block_list[*list_pos + 1] != 0) + // entry in the list. If the end of the list has been + // reached, reuse the size and filters of the last Block. + if (opt_block_list[*list_pos + 1].size != 0) { ++*list_pos; - *block_remaining = opt_block_list[*list_pos]; + // Update the filters if needed. + if (opt_block_list[*list_pos - 1].chain_num + != opt_block_list[*list_pos].chain_num) { + const unsigned chain_num + = opt_block_list[*list_pos].chain_num; + const lzma_filter *next = chains[chain_num]; + const lzma_ret ret = lzma_filters_update( + &strm, next); + + if (ret != LZMA_OK) { + // This message is only possible if + // the filter chain has unsupported + // options since the filter chain is + // validated using + // lzma_raw_encoder_memusage() or + // lzma_stream_encoder_mt_memusage(). + // Some options are not validated until + // the encoders are initialized. + message_fatal( + _("Error changing to " + "filter chain %u: %s"), + chain_num, + message_strm(ret)); + } + } + } + + *block_remaining = opt_block_list[*list_pos].size; // If in single-threaded mode, split up the Block if needed. // This is not needed in multi-threaded mode because liblzma @@ -764,6 +1106,7 @@ split_block(uint64_t *block_remaining, } } } +#endif static bool @@ -796,6 +1139,7 @@ coder_normal(file_pair *pair) // Assume that something goes wrong. bool success = false; +#ifdef HAVE_ENCODERS // block_remaining indicates how many input bytes to encode before // finishing the current .xz Block. The Block size is set with // --block-size=SIZE and --block-list. They have an effect only when @@ -829,15 +1173,18 @@ coder_normal(file_pair *pair) // output is still not identical because in single-threaded // mode the size info isn't written into Block Headers. if (opt_block_list != NULL) { - if (block_remaining < opt_block_list[list_pos]) { + if (block_remaining < opt_block_list[list_pos].size) { assert(!hardware_threads_is_mt()); - next_block_remaining = opt_block_list[list_pos] + next_block_remaining = + opt_block_list[list_pos].size - block_remaining; } else { - block_remaining = opt_block_list[list_pos]; + block_remaining = + opt_block_list[list_pos].size; } } } +#endif strm.next_out = out_buf.u8; strm.avail_out = IO_BUFFER_SIZE; @@ -847,17 +1194,22 @@ coder_normal(file_pair *pair) // flushing or finishing. if (strm.avail_in == 0 && action == LZMA_RUN) { strm.next_in = in_buf.u8; - strm.avail_in = io_read(pair, &in_buf, - my_min(block_remaining, - IO_BUFFER_SIZE)); +#ifdef HAVE_ENCODERS + const size_t read_size = my_min(block_remaining, + IO_BUFFER_SIZE); +#else + const size_t read_size = IO_BUFFER_SIZE; +#endif + strm.avail_in = io_read(pair, &in_buf, read_size); if (strm.avail_in == SIZE_MAX) break; if (pair->src_eof) { action = LZMA_FINISH; - - } else if (block_remaining != UINT64_MAX) { + } +#ifdef HAVE_ENCODERS + else if (block_remaining != UINT64_MAX) { // Start a new Block after every // opt_block_size bytes of input. block_remaining -= strm.avail_in; @@ -867,6 +1219,7 @@ coder_normal(file_pair *pair) if (action == LZMA_RUN && pair->flush_needed) action = LZMA_SYNC_FLUSH; +#endif } // Let liblzma do the actual work. @@ -878,6 +1231,7 @@ coder_normal(file_pair *pair) break; } +#ifdef HAVE_ENCODERS if (ret == LZMA_STREAM_END && (action == LZMA_SYNC_FLUSH || action == LZMA_FULL_BARRIER)) { if (action == LZMA_SYNC_FLUSH) { @@ -907,8 +1261,9 @@ coder_normal(file_pair *pair) // Start a new Block after LZMA_FULL_FLUSH or continue // the same block after LZMA_SYNC_FLUSH. action = LZMA_RUN; - - } else if (ret != LZMA_OK) { + } else +#endif + if (ret != LZMA_OK) { // Determine if the return value indicates that we // won't continue coding. LZMA_NO_CHECK would be // here too if LZMA_TELL_ANY_CHECK was used. @@ -1103,6 +1458,16 @@ coder_run(const char *filename) extern void coder_free(void) { + // Free starting from the second filter chain since the default + // filter chain may have its options set from a static variable + // in coder_set_compression_settings(). Since this is only run in + // debug mode and will be freed when the process ends anyway, we + // don't worry about freeing it. + for (uint32_t i = 1; i < ARRAY_SIZE(chains); i++) { + if (chains_used_mask & (1U << i)) + lzma_filters_free(chains[i], NULL); + } + lzma_end(&strm); return; } diff --git a/src/xz/coder.h b/src/xz/coder.h index b4f43a2..96755f3 100644 --- a/src/xz/coder.h +++ b/src/xz/coder.h @@ -1,12 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file coder.h /// \brief Compresses or uncompresses a file // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -30,6 +30,16 @@ enum format_type { }; +/// Array of these hold the entries specified with --block-list. +typedef struct { + /// Uncompressed size of the Block + uint64_t size; + + /// Filter chain to use for this Block (chains[chain_num]) + unsigned chain_num; +} block_list_entry; + + /// Operation mode of the command line tool. This is set in args.c and read /// in several files. extern enum operation_mode opt_mode; @@ -50,9 +60,21 @@ extern bool opt_single_stream; /// of input. This has an effect only when compressing to the .xz format. extern uint64_t opt_block_size; -/// This is non-NULL if --block-list was used. This contains the Block sizes -/// as an array that is terminated with 0. -extern uint64_t *opt_block_list; +/// List of block size and filter chain pointer pairs. +extern block_list_entry *opt_block_list; + +/// Size of the largest Block that was specified in --block-list. +/// This is used to limit the block_size option of multithreaded encoder. +/// It's waste of memory to specify a too large block_size and reducing +/// it might even allow using more threads in some cases. +/// +/// NOTE: If the last entry in --block-list is the special value of 0 +/// (which gets converted to UINT64_MAX), it counts here as UINT64_MAX too. +/// This way the multithreaded encoder's Block size won't be reduced. +extern uint64_t block_list_largest; + +/// Bitmask indicating which filter chains we specified in --block-list. +extern uint32_t block_list_chain_mask; /// Set the integrity check type used when compressing extern void coder_set_check(lzma_check check); @@ -77,3 +99,9 @@ extern void coder_run(const char *filename); /// Free the memory allocated for the coder and kill the worker threads. extern void coder_free(void); #endif + +/// Create filter chain from string +extern void coder_add_filters_from_str(const char *filter_str); + +/// Add or overwrite a filter that can be used by the block-list. +extern void coder_add_block_filters(const char *str, size_t slot); diff --git a/src/xz/file_io.c b/src/xz/file_io.c index 2828029..678a9a5 100644 --- a/src/xz/file_io.c +++ b/src/xz/file_io.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file file_io.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -29,15 +28,33 @@ static bool warn_fchown; # include <utime.h> #endif -#ifdef HAVE_CAPSICUM -# ifdef HAVE_SYS_CAPSICUM_H -# include <sys/capsicum.h> +#include "tuklib_open_stdxxx.h" + +#ifdef _MSC_VER +# ifdef _WIN64 + typedef __int64 ssize_t; # else -# include <sys/capability.h> + typedef int ssize_t; # endif -#endif -#include "tuklib_open_stdxxx.h" + typedef int mode_t; +# define S_IRUSR _S_IREAD +# define S_IWUSR _S_IWRITE + +# define setmode _setmode +# define open _open +# define close _close +# define lseek _lseeki64 +# define unlink _unlink + + // The casts are to silence warnings. + // The sizes are known to be small enough. +# define read(fd, buf, size) _read(fd, buf, (unsigned int)(size)) +# define write(fd, buf, size) _write(fd, buf, (unsigned int)(size)) + +# define S_ISDIR(m) (((m) & _S_IFMT) == _S_IFDIR) +# define S_ISREG(m) (((m) & _S_IFMT) == _S_IFREG) +#endif #ifndef O_BINARY # define O_BINARY 0 @@ -66,11 +83,6 @@ typedef enum { /// If true, try to create sparse files when decompressing. static bool try_sparse = true; -#ifdef ENABLE_SANDBOX -/// True if the conditions for sandboxing (described in main()) have been met. -static bool sandbox_allowed = false; -#endif - #ifndef TUKLIB_DOSLIKE /// File status flags of standard input. This is used by io_open_src() /// and io_close_src(). @@ -155,105 +167,6 @@ io_no_sparse(void) } -#ifdef ENABLE_SANDBOX -extern void -io_allow_sandbox(void) -{ - sandbox_allowed = true; - return; -} - - -/// Enables operating-system-specific sandbox if it is possible. -/// src_fd is the file descriptor of the input file. -static void -io_sandbox_enter(int src_fd) -{ - if (!sandbox_allowed) { - // This message is more often annoying than useful so - // it's commented out. It can be useful when developing - // the sandboxing code. - //message(V_DEBUG, _("Sandbox is disabled due " - // "to incompatible command line arguments")); - return; - } - - const char dummy_str[] = "x"; - - // Try to ensure that both libc and xz locale files have been - // loaded when NLS is enabled. - snprintf(NULL, 0, "%s%s", _(dummy_str), strerror(EINVAL)); - - // Try to ensure that iconv data files needed for handling multibyte - // characters have been loaded. This is needed at least with glibc. - tuklib_mbstr_width(dummy_str, NULL); - -#ifdef HAVE_CAPSICUM - // Capsicum needs FreeBSD 10.0 or later. - cap_rights_t rights; - - if (cap_enter()) - goto error; - - if (cap_rights_limit(src_fd, cap_rights_init(&rights, - CAP_EVENT, CAP_FCNTL, CAP_LOOKUP, CAP_READ, CAP_SEEK))) - goto error; - - if (src_fd != STDIN_FILENO && cap_rights_limit( - STDIN_FILENO, cap_rights_clear(&rights))) - goto error; - - if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights, - CAP_EVENT, CAP_FCNTL, CAP_FSTAT, CAP_LOOKUP, - CAP_WRITE, CAP_SEEK))) - goto error; - - if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights, - CAP_WRITE))) - goto error; - - if (cap_rights_limit(user_abort_pipe[0], cap_rights_init(&rights, - CAP_EVENT))) - goto error; - - if (cap_rights_limit(user_abort_pipe[1], cap_rights_init(&rights, - CAP_WRITE))) - goto error; - -#elif defined(HAVE_PLEDGE) - // pledge() was introduced in OpenBSD 5.9. - // - // main() unconditionally calls pledge() with fairly relaxed - // promises which work in all situations. Here we make the - // sandbox more strict. - if (pledge("stdio", "")) - goto error; - - (void)src_fd; - -#else -# error ENABLE_SANDBOX is defined but no sandboxing method was found. -#endif - - // This message is annoying in xz -lvv. - //message(V_DEBUG, _("Sandbox was successfully enabled")); - return; - -error: -#ifdef HAVE_CAPSICUM - // If a kernel is configured without capability mode support or - // used in an emulator that does not implement the capability - // system calls, then the Capsicum system calls will fail and set - // errno to ENOSYS. In that case xz will silently run without - // the sandbox. - if (errno == ENOSYS) - return; -#endif - message_fatal(_("Failed to enable the sandbox")); -} -#endif // ENABLE_SANDBOX - - #ifndef TUKLIB_DOSLIKE /// \brief Waits for input or output to become available or for a signal /// @@ -407,7 +320,7 @@ io_copy_attrs(const file_pair *pair) message_warning(_("%s: Cannot set the file group: %s"), pair->dest_name, strerror(errno)); // We can still safely copy some additional permissions: - // `group' must be at least as strict as `other' and + // 'group' must be at least as strict as 'other' and // also vice versa. // // NOTE: After this, the owner of the source file may @@ -809,7 +722,8 @@ io_open_src(const char *src_name) #ifdef ENABLE_SANDBOX if (!error) - io_sandbox_enter(pair.src_fd); + sandbox_enable_strict_if_allowed(pair.src_fd, + user_abort_pipe[0], user_abort_pipe[1]); #endif return error ? NULL : &pair; diff --git a/src/xz/file_io.h b/src/xz/file_io.h index 6992efa..ae7e2f3 100644 --- a/src/xz/file_io.h +++ b/src/xz/file_io.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file file_io.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// // Some systems have suboptimal BUFSIZ. Use a bit bigger value on them. @@ -18,6 +17,16 @@ # define IO_BUFFER_SIZE (BUFSIZ & ~7U) #endif +#ifdef _MSC_VER + // The first one renames both "struct stat" -> "struct _stat64" + // and stat() -> _stat64(). The documentation mentions only + // "struct __stat64", not "struct _stat64", but the latter + // works too. +# define stat _stat64 +# define fstat _fstat64 +# define off_t __int64 +#endif + /// is_sparse() accesses the buffer as uint64_t for maximum speed. /// The u32 and u64 members must only be access through this union @@ -90,12 +99,6 @@ extern void io_write_to_user_abort_pipe(void); extern void io_no_sparse(void); -#ifdef ENABLE_SANDBOX -/// \brief main() calls this if conditions for sandboxing have been met. -extern void io_allow_sandbox(void); -#endif - - /// \brief Open the source file extern file_pair *io_open_src(const char *src_name); diff --git a/src/xz/hardware.c b/src/xz/hardware.c index c694882..952652f 100644 --- a/src/xz/hardware.c +++ b/src/xz/hardware.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -15,7 +14,7 @@ /// Maximum number of worker threads. This can be set with /// the --threads=NUM command line option. -static uint32_t threads_max = 1; +static uint32_t threads_max; /// True when the number of threads is automatically determined based /// on the available hardware threads. @@ -334,5 +333,9 @@ hardware_init(void) memlimit_mt_default = mem_ceiling; #endif + // Enable threaded mode by default. xz 5.4.x and older + // used single-threaded mode by default. + hardware_threads_set(0); + return; } diff --git a/src/xz/hardware.h b/src/xz/hardware.h index a67b26e..25b351e 100644 --- a/src/xz/hardware.h +++ b/src/xz/hardware.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file hardware.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// Initialize some hardware-specific variables, which are needed by other diff --git a/src/xz/list.c b/src/xz/list.c index 86c3a76..ca9cf03 100644 --- a/src/xz/list.c +++ b/src/xz/list.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file list.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -544,11 +543,21 @@ parse_block_header(file_pair *pair, const lzma_index_iter *iter, xfi->memusage_max = bhi->memusage; // Determine the minimum XZ Utils version that supports this Block. + // - RISC-V filter needs 5.6.0. // // - ARM64 filter needs 5.4.0. // // - 5.0.0 doesn't support empty LZMA2 streams and thus empty // Blocks that use LZMA2. This decoder bug was fixed in 5.0.2. + if (xfi->min_version < 50060002U) { + for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { + if (filters[i].id == LZMA_FILTER_RISCV) { + xfi->min_version = 50060002U; + break; + } + } + } + if (xfi->min_version < 50040002U) { for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) { if (filters[i].id == LZMA_FILTER_ARM64) { @@ -1266,10 +1275,22 @@ list_totals(void) extern void list_file(const char *filename) { - if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO) - message_fatal(_("--list works only on .xz files " + if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO) { + // The 'lzmainfo' message is printed only when --format=lzma + // is used (it is implied if using "lzma" as the command + // name). Thus instead of using message_fatal(), print + // the messages separately and then call tuklib_exit() + // like message_fatal() does. + message(V_ERROR, _("--list works only on .xz files " "(--format=xz or --format=auto)")); + if (opt_format == FORMAT_LZMA) + message(V_ERROR, + _("Try 'lzmainfo' with .lzma files.")); + + tuklib_exit(E_ERROR, E_ERROR, false); + } + message_filename(filename); if (filename == stdin_filename) { diff --git a/src/xz/list.h b/src/xz/list.h index a4c6ec7..805880d 100644 --- a/src/xz/list.h +++ b/src/xz/list.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file list.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// \brief List information about the given .xz file diff --git a/src/xz/main.c b/src/xz/main.c index c9c3dec..71b5ef7 100644 --- a/src/xz/main.c +++ b/src/xz/main.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file main.c @@ -5,14 +7,12 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" #include <ctype.h> + /// Exit status to use. This can be changed with set_exit_status(). static enum exit_status_type exit_status = E_SUCCESS; @@ -119,8 +119,8 @@ read_name(const args_info *args) // newlines. message_error(_("%s: Null character found when " "reading filenames; maybe you meant " - "to use `--files0' instead " - "of `--files'?"), args->files_name); + "to use '--files0' instead " + "of '--files'?"), args->files_name); return NULL; } @@ -142,35 +142,38 @@ read_name(const args_info *args) int main(int argc, char **argv) { -#ifdef HAVE_PLEDGE - // OpenBSD's pledge(2) sandbox - // - // Unconditionally enable sandboxing with fairly relaxed promises. - // This is still way better than having no sandbox at all. :-) - // More strict promises will be made later in file_io.c if possible. - if (pledge("stdio rpath wpath cpath fattr", "")) { - // Don't translate the string or use message_fatal() as - // those haven't been initialized yet. - fprintf(stderr, "%s: Failed to enable the sandbox\n", argv[0]); - return E_ERROR; - } -#endif - #if defined(_WIN32) && !defined(__CYGWIN__) InitializeCriticalSection(&exit_status_cs); #endif - // Set up the progname variable. + // Set up the progname variable needed for messages. tuklib_progname_init(argv); // Initialize the file I/O. This makes sure that // stdin, stdout, and stderr are something valid. + // This must be done before we might open any files + // even indirectly like locale and gettext initializations. io_init(); +#ifdef ENABLE_SANDBOX + // Enable such sandboxing that can always be enabled. + // This requires that progname has been set up. + // It's also good that io_init() has been called because it + // might need to do things that the initial sandbox won't allow. + // Otherwise this should be called as early as possible. + // + // NOTE: Calling this before tuklib_gettext_init() means that + // translated error message won't be available if sandbox + // initialization fails. However, sandbox_init() shouldn't + // fail and this order simply feels better. + sandbox_init(); +#endif + // Set up the locale and message translations. tuklib_gettext_init(PACKAGE, LOCALEDIR); - // Initialize handling of error/warning/other messages. + // Initialize progress message handling. It's not always needed + // but it's simpler to do this unconditionally. message_init(); // Set hardware-dependent default values. These can be overridden @@ -220,21 +223,41 @@ main(int argc, char **argv) signals_init(); #ifdef ENABLE_SANDBOX - // Set a flag that sandboxing is allowed if all these are true: - // - --files or --files0 wasn't used. - // - There is exactly one input file or we are reading from stdin. - // - We won't create any files: output goes to stdout or --test - // or --list was used. Note that --test implies opt_stdout = true - // but --list doesn't. + // Read-only sandbox can be enabled if we won't create or delete + // any files: // - // This is obviously not ideal but it was easy to implement and - // it covers the most common use cases. + // - --stdout, --test, or --list was used. Note that --test + // implies opt_stdout = true but --list doesn't. // - // TODO: Make sandboxing work for other situations too. - if (args.files_name == NULL && args.arg_count == 1 - && (opt_stdout || strcmp("-", args.arg_names[0]) == 0 - || opt_mode == MODE_LIST)) - io_allow_sandbox(); + // - Output goes to stdout because --files or --files0 wasn't used + // and no arguments were given on the command line or the + // arguments are all "-" (indicating standard input). + bool to_stdout_only = opt_stdout || opt_mode == MODE_LIST; + if (!to_stdout_only && args.files_name == NULL) { + // If all of the filenames provided are "-" (more than one + // "-" could be specified), then we are only going to be + // writing to standard output. Note that if no filename args + // were provided, args.c puts a single "-" in arg_names[0]. + to_stdout_only = true; + + for (unsigned i = 0; i < args.arg_count; ++i) { + if (strcmp("-", args.arg_names[i]) != 0) { + to_stdout_only = false; + break; + } + } + } + + if (to_stdout_only) { + sandbox_enable_read_only(); + + // Allow strict sandboxing if we are processing exactly one + // file to standard output. This requires that --files or + // --files0 wasn't specified (an unknown number of filenames + // could be provided that way). + if (args.files_name == NULL && args.arg_count == 1) + sandbox_allow_strict(); + } #endif // coder_run() handles compression, decompression, and testing. diff --git a/src/xz/main.h b/src/xz/main.h index 323f2f7..a8a1b45 100644 --- a/src/xz/main.h +++ b/src/xz/main.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file main.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// Possible exit status values. These are the same as used by gzip and bzip2. diff --git a/src/xz/message.c b/src/xz/message.c index abf30ad..deafdb4 100644 --- a/src/xz/message.c +++ b/src/xz/message.c @@ -1,12 +1,12 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file message.c /// \brief Printing messages // -// Author: Lasse Collin -// -// This file has been put into the public domain. -// You can do whatever you want with this file. +// Authors: Lasse Collin +// Jia Tan // /////////////////////////////////////////////////////////////////////////////// @@ -42,7 +42,7 @@ static bool current_filename_printed = false; /// True if we should print progress indicator and update it automatically /// if also verbose >= V_VERBOSE. -static bool progress_automatic; +static bool progress_automatic = false; /// True if message_progress_start() has been called but /// message_progress_end() hasn't been called yet. @@ -119,26 +119,7 @@ message_init(void) // exception, even if --verbose was not used, user can send SIGALRM // to make us print progress information once without automatic // updating. - progress_automatic = isatty(STDERR_FILENO); - - // Commented out because COLUMNS is rarely exported to environment. - // Most users have at least 80 columns anyway, let's think something - // fancy here if enough people complain. -/* - if (progress_automatic) { - // stderr is a terminal. Check the COLUMNS environment - // variable to see if the terminal is wide enough. If COLUMNS - // doesn't exist or it has some unparsable value, we assume - // that the terminal is wide enough. - const char *columns_str = getenv("COLUMNS"); - if (columns_str != NULL) { - char *endptr; - const long columns = strtol(columns_str, &endptr, 10); - if (*endptr != '\0' || columns < 80) - progress_automatic = false; - } - } -*/ + progress_automatic = is_tty(STDERR_FILENO); #ifdef SIGALRM // Establish the signal handlers which set a flag to tell us that @@ -932,7 +913,7 @@ message_try_help(void) { // Print this with V_WARNING instead of V_ERROR to prevent it from // showing up when --quiet has been specified. - message(V_WARNING, _("Try `%s --help' for more information."), + message(V_WARNING, _("Try '%s --help' for more information."), progname); return; } @@ -994,7 +975,7 @@ message_help(bool long_help) " ignore possible remaining input data")); puts(_( " --no-sparse do not create sparse files when decompressing\n" -" -S, --suffix=.SUF use the suffix `.SUF' on compressed files\n" +" -S, --suffix=.SUF use the suffix '.SUF' on compressed files\n" " --files[=FILE] read filenames to process from FILE; if FILE is\n" " omitted, filenames are read from the standard input;\n" " filenames must be terminated with the newline character\n" @@ -1005,9 +986,9 @@ message_help(bool long_help) puts(_("\n Basic file format and compression options:\n")); puts(_( " -F, --format=FMT file format to encode or decode; possible values are\n" -" `auto' (default), `xz', `lzma', `lzip', and `raw'\n" -" -C, --check=CHECK integrity check type: `none' (use with caution),\n" -" `crc32', `crc64' (default), or `sha256'")); +" 'auto' (default), 'xz', 'lzma', 'lzip', and 'raw'\n" +" -C, --check=CHECK integrity check type: 'none' (use with caution),\n" +" 'crc32', 'crc64' (default), or 'sha256'")); puts(_( " --ignore-check don't verify the integrity check when decompressing")); } @@ -1021,8 +1002,8 @@ message_help(bool long_help) " does not affect decompressor memory requirements")); puts(_( -" -T, --threads=NUM use at most NUM threads; the default is 1; set to 0\n" -" to use as many threads as there are processor cores")); +" -T, --threads=NUM use at most NUM threads; the default is 0 which uses\n" +" as many threads as there are processor cores")); if (long_help) { puts(_( @@ -1030,9 +1011,11 @@ message_help(bool long_help) " start a new .xz block after every SIZE bytes of input;\n" " use this to set the block size for threaded compression")); puts(_( -" --block-list=SIZES\n" +" --block-list=BLOCKS\n" " start a new .xz block after the given comma-separated\n" -" intervals of uncompressed data")); +" intervals of uncompressed data; optionally, specify a\n" +" filter chain number (0-9) followed by a ':' before the\n" +" uncompressed data size")); puts(_( " --flush-timeout=TIMEOUT\n" " when compressing, if more than TIMEOUT milliseconds has\n" @@ -1057,6 +1040,23 @@ message_help(bool long_help) puts(_( "\n Custom filter chain for compression (alternative for using presets):")); + puts(_( +"\n" +" --filters=FILTERS set the filter chain using the liblzma filter string\n" +" syntax; use --filters-help for more information" + )); + + puts(_( +" --filters1=FILTERS ... --filters9=FILTERS\n" +" set additional filter chains using the liblzma filter\n" +" string syntax to use with --block-list" + )); + + puts(_( +" --filters-help display more information about the liblzma filter string\n" +" syntax and exit." + )); + #if defined(HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) \ || defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2) // TRANSLATORS: The word "literal" in "literal context bits" @@ -1087,6 +1087,7 @@ message_help(bool long_help) " --powerpc[=OPTS] PowerPC BCJ filter (big endian only)\n" " --ia64[=OPTS] IA-64 (Itanium) BCJ filter\n" " --sparc[=OPTS] SPARC BCJ filter\n" +" --riscv[=OPTS] RISC-V BCJ filter\n" " Valid OPTS for all BCJ filters:\n" " start=NUM start offset for conversions (default=0)")); @@ -1144,3 +1145,28 @@ message_help(bool long_help) tuklib_exit(E_SUCCESS, E_ERROR, verbosity != V_SILENT); } + + +extern void +message_filters_help(void) +{ + char *encoder_options; + if (lzma_str_list_filters(&encoder_options, LZMA_VLI_UNKNOWN, + LZMA_STR_ENCODER, NULL) != LZMA_OK) + message_bug(); + + if (!opt_robot) { + puts(_( +"Filter chains are set using the --filters=FILTERS or\n" +"--filters1=FILTERS ... --filters9=FILTERS options. Each filter in the chain\n" +"can be separated by spaces or '--'. Alternatively a preset <0-9>[e] can be\n" +"specified instead of a filter chain.\n" + )); + + puts(_("The supported filters and their options are:")); + } + + puts(encoder_options); + + tuklib_exit(E_SUCCESS, E_ERROR, verbosity != V_SILENT); +} diff --git a/src/xz/message.h b/src/xz/message.h index f608ec7..3f6e183 100644 --- a/src/xz/message.h +++ b/src/xz/message.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file message.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// Verbosity levels @@ -24,7 +23,10 @@ enum message_verbosity { extern const int message_progress_sigs[]; -/// \brief Initializes the message functions +/// \brief Initializes the progress message functions +/// +/// message_fatal() and such can be called even before message_init() +/// has been called. /// /// If an error occurs, this function doesn't return. /// @@ -111,6 +113,12 @@ tuklib_attr_noreturn extern void message_help(bool long_help); +/// Prints a help message specifically for using the --filters and +/// --filtersX command line options. +tuklib_attr_noreturn +extern void message_filters_help(void); + + /// \brief Set the total number of files to be processed /// /// Standard input is counted as a file here. This is used when printing diff --git a/src/xz/mytime.c b/src/xz/mytime.c index 7e8a074..7d9a27d 100644 --- a/src/xz/mytime.c +++ b/src/xz/mytime.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file mytime.c @@ -5,14 +7,14 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" -#if defined(HAVE_CLOCK_GETTIME) && defined(HAVE_CLOCK_MONOTONIC) +#if defined(MYTHREAD_VISTA) || defined(_MSC_VER) + // Nothing +#elif defined(HAVE_CLOCK_GETTIME) \ + && (!defined(__MINGW32__) || defined(MYTHREAD_POSIX)) # include <time.h> #else # include <sys/time.h> @@ -20,7 +22,22 @@ uint64_t opt_flush_timeout = 0; +// start_time holds the time when the (de)compression was started. +// It's from mytime_now() and thus only useful for calculating relative +// time differences (elapsed time). start_time is initialized by calling +// mytime_set_start_time() and modified by mytime_sigtstp_handler(). +// +// When mytime_sigtstp_handler() is used, start_time is made volatile. +// I'm not sure if that is really required since access to it is guarded +// by signals_block()/signals_unblock() since accessing an uint64_t isn't +// atomic on all systems. But since the variable isn't accessed very +// frequently making it volatile doesn't hurt. +#ifdef USE_SIGTSTP_HANDLER +static volatile uint64_t start_time; +#else static uint64_t start_time; +#endif + static uint64_t next_flush; @@ -30,16 +47,41 @@ static uint64_t next_flush; static uint64_t mytime_now(void) { -#if defined(HAVE_CLOCK_GETTIME) && defined(HAVE_CLOCK_MONOTONIC) +#if defined(MYTHREAD_VISTA) || defined(_MSC_VER) + // Since there is no SIGALRM on Windows, this function gets + // called frequently when the progress indicator is in use. + // Progress indicator doesn't need high-resolution time. + // GetTickCount64() has very low overhead but needs at least WinVista. + // + // MinGW-w64 provides the POSIX functions clock_gettime() and + // gettimeofday() in a manner that allow xz to run on older + // than WinVista. If the threading method needs WinVista anyway, + // there's no reason to avoid a WinVista API here either. + return GetTickCount64(); + +#elif defined(HAVE_CLOCK_GETTIME) \ + && (!defined(__MINGW32__) || defined(MYTHREAD_POSIX)) + // MinGW-w64: clock_gettime() is defined in winpthreads but we need + // nothing else from winpthreads (unless, for some odd reason, POSIX + // threading has been selected). By avoiding clock_gettime(), we + // avoid the dependency on libwinpthread-1.dll or the need to link + // against the static version. The downside is that the fallback + // method, gettimeofday(), doesn't provide monotonic time. + struct timespec tv; + +# ifdef HAVE_CLOCK_MONOTONIC // If CLOCK_MONOTONIC was available at compile time but for some // reason isn't at runtime, fallback to CLOCK_REALTIME which // according to POSIX is mandatory for all implementations. static clockid_t clk_id = CLOCK_MONOTONIC; - struct timespec tv; while (clock_gettime(clk_id, &tv)) clk_id = CLOCK_REALTIME; +# else + clock_gettime(CLOCK_REALTIME, &tv); +# endif return (uint64_t)tv.tv_sec * 1000 + (uint64_t)(tv.tv_nsec / 1000000); + #else struct timeval tv; gettimeofday(&tv, NULL); @@ -48,10 +90,49 @@ mytime_now(void) } +#ifdef USE_SIGTSTP_HANDLER +extern void +mytime_sigtstp_handler(int sig lzma_attribute((__unused__))) +{ + // Measure how long the process stays in the stopped state and add + // that amount to start_time. This way the the progress indicator + // won't count the stopped time as elapsed time and the estimated + // remaining time won't be confused by the time spent in the + // stopped state. + // + // FIXME? Is raising SIGSTOP the correct thing to do? POSIX.1-2017 + // says that orphan processes shouldn't stop on SIGTSTP. So perhaps + // the most correct thing to do could be to revert to the default + // handler for SIGTSTP, unblock SIGTSTP, and then raise(SIGTSTP). + // It's quite a bit more complicated than just raising SIGSTOP though. + // + // The difference between raising SIGTSTP vs. SIGSTOP can be seen on + // the shell command line too by running "echo $?" after stopping + // a process but perhaps that doesn't matter. + const uint64_t t = mytime_now(); + raise(SIGSTOP); + start_time += mytime_now() - t; + return; +} +#endif + + extern void mytime_set_start_time(void) { +#ifdef USE_SIGTSTP_HANDLER + // Block the signals when accessing start_time so that we cannot + // end up with a garbage value. start_time is volatile but access + // to it isn't atomic at least on 32-bit systems. + signals_block(); +#endif + start_time = mytime_now(); + +#ifdef USE_SIGTSTP_HANDLER + signals_unblock(); +#endif + return; } @@ -59,7 +140,17 @@ mytime_set_start_time(void) extern uint64_t mytime_get_elapsed(void) { - return mytime_now() - start_time; +#ifdef USE_SIGTSTP_HANDLER + signals_block(); +#endif + + const uint64_t t = mytime_now() - start_time; + +#ifdef USE_SIGTSTP_HANDLER + signals_unblock(); +#endif + + return t; } diff --git a/src/xz/mytime.h b/src/xz/mytime.h index a7be2aa..6dfaeae 100644 --- a/src/xz/mytime.h +++ b/src/xz/mytime.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file mytime.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// @@ -21,6 +20,12 @@ extern uint64_t opt_flush_timeout; +#ifdef USE_SIGTSTP_HANDLER +/// \brief Signal handler for SIGTSTP +extern void mytime_sigtstp_handler(int sig); +#endif + + /// \brief Store the time when (de)compression was started /// /// The start time is also stored as the time of the first flush. diff --git a/src/xz/options.c b/src/xz/options.c index 4d5e899..bc8bc1a 100644 --- a/src/xz/options.c +++ b/src/xz/options.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file options.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -83,7 +82,7 @@ parse_options(const char *str, const option_map *opts, *value++ = '\0'; if (value == NULL || value[0] == '\0') - message_fatal(_("%s: Options must be `name=value' " + message_fatal(_("%s: Options must be 'name=value' " "pairs separated with commas"), str); // Look for the option name from the option map. diff --git a/src/xz/options.h b/src/xz/options.h index 61ec8d5..4a1314d 100644 --- a/src/xz/options.h +++ b/src/xz/options.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file options.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// \brief Parser for Delta options diff --git a/src/xz/private.h b/src/xz/private.h index 6414bdb..b370472 100644 --- a/src/xz/private.h +++ b/src/xz/private.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file private.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "sysdefs.h" @@ -21,7 +20,10 @@ #include <signal.h> #include <locale.h> #include <stdio.h> -#include <unistd.h> + +#ifndef _MSC_VER +# include <unistd.h> +#endif #include "tuklib_gettext.h" #include "tuklib_progname.h" @@ -33,6 +35,10 @@ # include <windows.h> #endif +#ifdef _MSC_VER +# define fileno _fileno +#endif + #ifndef STDIN_FILENO # define STDIN_FILENO (fileno(stdin)) #endif @@ -45,8 +51,15 @@ # define STDERR_FILENO (fileno(stderr)) #endif -#if defined(HAVE_CAPSICUM) || defined(HAVE_PLEDGE) -# define ENABLE_SANDBOX 1 +// Handling SIGTSTP keeps time-keeping for progress indicator correct +// if xz is stopped. It requires use of clock_gettime() as that is +// async-signal safe in POSIX. Require also SIGALRM support since +// on systems where SIGALRM isn't available, progress indicator code +// polls the time and the SIGTSTP handling adds slight overhead to +// that code. Most (all?) systems that have SIGTSTP also have SIGALRM +// so this requirement won't exclude many systems. +#if defined(HAVE_CLOCK_GETTIME) && defined(SIGTSTP) && defined(SIGALRM) +# define USE_SIGTSTP_HANDLER 1 #endif #include "main.h" @@ -57,6 +70,7 @@ #include "hardware.h" #include "file_io.h" #include "options.h" +#include "sandbox.h" #include "signals.h" #include "suffix.h" #include "util.h" diff --git a/src/xz/sandbox.c b/src/xz/sandbox.c new file mode 100644 index 0000000..5bd2273 --- /dev/null +++ b/src/xz/sandbox.c @@ -0,0 +1,355 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file sandbox.c +/// \brief Sandbox support +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#include "private.h" + + +#ifndef ENABLE_SANDBOX + +// Prevent an empty translation unit when no sandboxing is supported. +typedef int dummy; + +#else + +/// If the conditions for strict sandboxing (described in main()) +/// have been met, sandbox_allow_strict() can be called to set this +/// variable to true. +static bool strict_sandbox_allowed = false; + + +extern void +sandbox_allow_strict(void) +{ + strict_sandbox_allowed = true; + return; +} + + +// Strict sandboxing prevents opening any files. This *tries* to ensure +// that any auxiliary files that might be required are already open. +// +// Returns true if strict sandboxing is allowed, false otherwise. +static bool +prepare_for_strict_sandbox(void) +{ + if (!strict_sandbox_allowed) + return false; + + const char dummy_str[] = "x"; + + // Try to ensure that both libc and xz locale files have been + // loaded when NLS is enabled. + snprintf(NULL, 0, "%s%s", _(dummy_str), strerror(EINVAL)); + + // Try to ensure that iconv data files needed for handling multibyte + // characters have been loaded. This is needed at least with glibc. + tuklib_mbstr_width(dummy_str, NULL); + + return true; +} + +#endif + + +#if defined(HAVE_PLEDGE) + +/////////////// +// pledge(2) // +/////////////// + +#include <unistd.h> + + +extern void +sandbox_init(void) +{ + if (pledge("stdio rpath wpath cpath fattr", "")) { + // gettext hasn't been initialized yet so + // there's no point to call it here. + message_fatal("Failed to enable the sandbox"); + } + + return; +} + + +extern void +sandbox_enable_read_only(void) +{ + // We will be opening files for reading but + // won't create or remove any files. + if (pledge("stdio rpath", "")) + message_fatal(_("Failed to enable the sandbox")); + + return; +} + + +extern void +sandbox_enable_strict_if_allowed(int src_fd lzma_attribute((__unused__)), + int pipe_event_fd lzma_attribute((__unused__)), + int pipe_write_fd lzma_attribute((__unused__))) +{ + if (!prepare_for_strict_sandbox()) + return; + + // All files that need to be opened have already been opened. + if (pledge("stdio", "")) + message_fatal(_("Failed to enable the sandbox")); + + return; +} + + +#elif defined(HAVE_LINUX_LANDLOCK) + +////////////// +// Landlock // +////////////// + +#include <linux/landlock.h> +#include <sys/syscall.h> +#include <sys/prctl.h> + + +// Highest Landlock ABI version supported by this file: +// - For ABI versions 1-3 we don't need anything from <linux/landlock.h> +// that isn't part of version 1. +// - For ABI version 4 we need the larger struct landlock_ruleset_attr +// with the handled_access_net member. That is bundled with the macros +// LANDLOCK_ACCESS_NET_BIND_TCP and LANDLOCK_ACCESS_NET_CONNECT_TCP. +#ifdef LANDLOCK_ACCESS_NET_BIND_TCP +# define LANDLOCK_ABI_MAX 4 +#else +# define LANDLOCK_ABI_MAX 3 +#endif + + +/// Landlock ABI version supported by the kernel +static int landlock_abi; + + +// The required_rights should have those bits set that must not be restricted. +// This function will then bitwise-and ~required_rights with a mask matching +// the Landlock ABI version, leaving only those bits set that are supported +// by the ABI and allowed to be restricted by the function argument. +static void +enable_landlock(uint64_t required_rights) +{ + assert(landlock_abi <= LANDLOCK_ABI_MAX); + + if (landlock_abi <= 0) + return; + + // We want to set all supported flags in handled_access_fs. + // This way the ruleset will initially forbid access to all + // actions that the available Landlock ABI version supports. + // Exceptions can be added using landlock_add_rule(2) to + // allow certain actions on certain files or directories. + // + // The same flag values are used on all archs. ABI v2 and v3 + // both add one new flag. + // + // First in ABI v1: LANDLOCK_ACCESS_FS_EXECUTE = 1ULL << 0 + // Last in ABI v1: LANDLOCK_ACCESS_FS_MAKE_SYM = 1ULL << 12 + // Last in ABI v2: LANDLOCK_ACCESS_FS_REFER = 1ULL << 13 + // Last in ABI v3: LANDLOCK_ACCESS_FS_TRUNCATE = 1ULL << 14 + // + // This makes it simple to set the mask based on the ABI + // version and we don't need to care which flags are #defined + // in the installed <linux/landlock.h> for ABI versions 1-3. + const struct landlock_ruleset_attr attr = { + .handled_access_fs = ~required_rights + & ((1ULL << (12 + my_min(3, landlock_abi))) - 1), +#if LANDLOCK_ABI_MAX >= 4 + .handled_access_net = landlock_abi < 4 ? 0 : + (LANDLOCK_ACCESS_NET_BIND_TCP + | LANDLOCK_ACCESS_NET_CONNECT_TCP), +#endif + }; + + const int ruleset_fd = syscall(SYS_landlock_create_ruleset, + &attr, sizeof(attr), 0U); + if (ruleset_fd < 0) + message_fatal(_("Failed to enable the sandbox")); + + // All files we need should have already been opened. Thus, + // we don't need to add any rules using landlock_add_rule(2) + // before activating the sandbox. + // + // NOTE: It's possible that the hack prepare_for_strict_sandbox() + // isn't be good enough. It tries to get translations and + // libc-specific files loaded but if it's not good enough + // then perhaps a Landlock rule to allow reading from /usr + // and/or the xz installation prefix would be needed. + // + // prctl(PR_SET_NO_NEW_PRIVS, ...) was already called in + // sandbox_init() so we don't do it here again. + if (syscall(SYS_landlock_restrict_self, ruleset_fd, 0U) != 0) + message_fatal(_("Failed to enable the sandbox")); + + return; +} + + +extern void +sandbox_init(void) +{ + // Prevent the process from gaining new privileges. This must be done + // before landlock_restrict_self(2) but since we will never need new + // privileges, this call can be done here already. + // + // This is supported since Linux 3.5. Ignore the return value to + // keep compatibility with old kernels. landlock_restrict_self(2) + // will fail if the no_new_privs attribute isn't set, thus if prctl() + // fails here the error will still be detected when it matters. + (void)prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); + + // Get the highest Landlock ABI version supported by the kernel. + landlock_abi = syscall(SYS_landlock_create_ruleset, + (void *)NULL, 0, LANDLOCK_CREATE_RULESET_VERSION); + + // The kernel might support a newer ABI than this file. + if (landlock_abi > LANDLOCK_ABI_MAX) + landlock_abi = LANDLOCK_ABI_MAX; + + // These are all in ABI version 1 already. We don't need truncate + // rights because files are created with open() using O_EXCL and + // without O_TRUNC. + // + // LANDLOCK_ACCESS_FS_READ_DIR is included here to get a clear error + // message if xz is given a directory name. Without this permission + // the message would be "Permission denied" but with this permission + // it's "Is a directory, skipping". It could be worked around with + // stat()/lstat() but just giving this permission is simpler and + // shouldn't make the sandbox much weaker in practice. + const uint64_t required_rights + = LANDLOCK_ACCESS_FS_WRITE_FILE + | LANDLOCK_ACCESS_FS_READ_FILE + | LANDLOCK_ACCESS_FS_READ_DIR + | LANDLOCK_ACCESS_FS_REMOVE_FILE + | LANDLOCK_ACCESS_FS_MAKE_REG; + + enable_landlock(required_rights); + return; +} + + +extern void +sandbox_enable_read_only(void) +{ + // We will be opening files for reading but + // won't create or remove any files. + const uint64_t required_rights + = LANDLOCK_ACCESS_FS_READ_FILE + | LANDLOCK_ACCESS_FS_READ_DIR; + enable_landlock(required_rights); + return; +} + + +extern void +sandbox_enable_strict_if_allowed(int src_fd lzma_attribute((__unused__)), + int pipe_event_fd lzma_attribute((__unused__)), + int pipe_write_fd lzma_attribute((__unused__))) +{ + if (!prepare_for_strict_sandbox()) + return; + + // Allow all restrictions that the kernel supports with the + // highest Landlock ABI version that the kernel or xz supports. + // + // NOTE: LANDLOCK_ACCESS_FS_READ_DIR isn't needed here because + // the only input file has already been opened. + enable_landlock(0); + return; +} + + +#elif defined(HAVE_CAP_RIGHTS_LIMIT) + +////////////// +// Capsicum // +////////////// + +#include <sys/capsicum.h> + + +extern void +sandbox_init(void) +{ + // Nothing to do. + return; +} + + +extern void +sandbox_enable_read_only(void) +{ + // Nothing to do. + return; +} + + +extern void +sandbox_enable_strict_if_allowed( + int src_fd, int pipe_event_fd, int pipe_write_fd) +{ + if (!prepare_for_strict_sandbox()) + return; + + // Capsicum needs FreeBSD 10.2 or later. + cap_rights_t rights; + + if (cap_enter()) + goto error; + + if (cap_rights_limit(src_fd, cap_rights_init(&rights, + CAP_EVENT, CAP_FCNTL, CAP_LOOKUP, CAP_READ, CAP_SEEK))) + goto error; + + // If not reading from stdin, remove all capabilities from it. + if (src_fd != STDIN_FILENO && cap_rights_limit( + STDIN_FILENO, cap_rights_clear(&rights))) + goto error; + + if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights, + CAP_EVENT, CAP_FCNTL, CAP_FSTAT, CAP_LOOKUP, + CAP_WRITE, CAP_SEEK))) + goto error; + + if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights, + CAP_WRITE))) + goto error; + + if (cap_rights_limit(pipe_event_fd, cap_rights_init(&rights, + CAP_EVENT))) + goto error; + + if (cap_rights_limit(pipe_write_fd, cap_rights_init(&rights, + CAP_WRITE))) + goto error; + + return; + +error: + // If a kernel is configured without capability mode support or + // used in an emulator that does not implement the capability + // system calls, then the Capsicum system calls will fail and set + // errno to ENOSYS. In that case xz will silently run without + // the sandbox. + if (errno == ENOSYS) + return; + + message_fatal(_("Failed to enable the sandbox")); +} + +#endif diff --git a/src/xz/sandbox.h b/src/xz/sandbox.h new file mode 100644 index 0000000..98b9862 --- /dev/null +++ b/src/xz/sandbox.h @@ -0,0 +1,43 @@ +// SPDX-License-Identifier: 0BSD + +/////////////////////////////////////////////////////////////////////////////// +// +/// \file sandbox.h +/// \brief Sandbox support +// +// Author: Lasse Collin +// +/////////////////////////////////////////////////////////////////////////////// + +#if defined(HAVE_PLEDGE) || defined(HAVE_LINUX_LANDLOCK) \ + || defined(HAVE_CAP_RIGHTS_LIMIT) +# define ENABLE_SANDBOX 1 +#endif + + +/// \brief Enables early sandboxing that can always be enabled +/// +/// This requires that tuklib_progname() and io_init() have been called. +extern void sandbox_init(void); + + +/// \brief Enable sandboxing that only allows opening files for reading +extern void sandbox_enable_read_only(void); + + +/// \brief Tell sandboxing code that strict sandboxing can be used +/// +/// This function only sets a flag which will be read by +/// sandbox_enable_strict_if_allowed(). +extern void sandbox_allow_strict(void); + + +/// \brief Enable sandboxing that allows reading from one file +/// +/// This does nothing if sandbox_allow_strict() hasn't been called. +/// +/// \param src_fd File descriptor open for reading +/// \param pipe_event_fd user_abort_pipe[0] from file_io.c +/// \param pipe_write_fd user_abort_pipe[1] from file_io.c +extern void sandbox_enable_strict_if_allowed( + int src_fd, int pipe_event_fd, int pipe_write_fd); diff --git a/src/xz/signals.c b/src/xz/signals.c index 7aef463..20f03be 100644 --- a/src/xz/signals.c +++ b/src/xz/signals.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file signals.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -50,6 +49,10 @@ signal_handler(int sig) } +#ifdef __APPLE__ +# pragma GCC diagnostic push +# pragma GCC diagnostic ignored "-Wsign-conversion" +#endif extern void signals_init(void) { @@ -82,6 +85,11 @@ signals_init(void) sigaddset(&hooked_signals, message_progress_sigs[i]); #endif +#ifdef USE_SIGTSTP_HANDLER + // Add the SIGTSTP handler from mytime.c to hooked_signals. + sigaddset(&hooked_signals, SIGTSTP); +#endif + // Using "my_sa" because "sa" may conflict with a sockaddr variable // from system headers on Solaris. struct sigaction my_sa; @@ -96,10 +104,11 @@ signals_init(void) my_sa.sa_flags = 0; my_sa.sa_handler = &signal_handler; + struct sigaction old; + for (size_t i = 0; i < ARRAY_SIZE(sigs); ++i) { // If the parent process has left some signals ignored, // we don't unignore them. - struct sigaction old; if (sigaction(sigs[i], NULL, &old) == 0 && old.sa_handler == SIG_IGN) continue; @@ -109,10 +118,22 @@ signals_init(void) message_signal_handler(); } +#ifdef USE_SIGTSTP_HANDLER + if (!(sigaction(SIGTSTP, NULL, &old) == 0 + && old.sa_handler == SIG_IGN)) { + my_sa.sa_handler = &mytime_sigtstp_handler; + if (sigaction(SIGTSTP, &my_sa, NULL)) + message_signal_handler(); + } +#endif + signals_are_initialized = true; return; } +#ifdef __APPLE__ +# pragma GCC diagnostic pop +#endif #ifndef __VMS diff --git a/src/xz/signals.h b/src/xz/signals.h index 5b125e0..629335d 100644 --- a/src/xz/signals.h +++ b/src/xz/signals.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file signals.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// If this is true, we will clean up the possibly incomplete output file, diff --git a/src/xz/suffix.c b/src/xz/suffix.c index 6ce9787..1d548e4 100644 --- a/src/xz/suffix.c +++ b/src/xz/suffix.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file suffix.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -21,7 +20,13 @@ # ifdef HAVE_STRINGS_H # include <strings.h> # endif -# define strcmp strcasecmp +# ifdef _MSC_VER +# define suffix_strcmp _stricmp +# else +# define suffix_strcmp strcasecmp +# endif +#else +# define suffix_strcmp strcmp #endif @@ -98,7 +103,7 @@ test_suffix(const char *suffix, const char *src_name, size_t src_len) || is_dir_sep(src_name[src_len - suffix_len - 1])) return 0; - if (strcmp(suffix, src_name + src_len - suffix_len) == 0) + if (suffix_strcmp(suffix, src_name + src_len - suffix_len) == 0) return src_len - suffix_len; return 0; @@ -178,7 +183,7 @@ uncompressed_name(const char *src_name, const size_t src_len) static void msg_suffix(const char *src_name, const char *suffix) { - message_warning(_("%s: File already has `%s' suffix, skipping"), + message_warning(_("%s: File already has '%s' suffix, skipping"), src_name, suffix); return; } diff --git a/src/xz/suffix.h b/src/xz/suffix.h index 135e905..f59e312 100644 --- a/src/xz/suffix.h +++ b/src/xz/suffix.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file suffix.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// \brief Get the name of the destination file diff --git a/src/xz/util.c b/src/xz/util.c index 6ab4c2d..0d339ae 100644 --- a/src/xz/util.c +++ b/src/xz/util.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file util.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "private.h" @@ -148,8 +147,8 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max) if (multiplier == 0) { message(V_ERROR, _("%s: Invalid multiplier suffix"), value - 1); - message_fatal(_("Valid suffixes are `KiB' (2^10), " - "`MiB' (2^20), and `GiB' (2^30).")); + message_fatal(_("Valid suffixes are 'KiB' (2^10), " + "'MiB' (2^20), and 'GiB' (2^30).")); } // Don't overflow here either. @@ -165,7 +164,7 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max) return result; error: - message_fatal(_("Value of the option `%s' must be in the range " + message_fatal(_("Value of the option '%s' must be in the range " "[%" PRIu64 ", %" PRIu64 "]"), name, min, max); } @@ -262,9 +261,30 @@ my_snprintf(char **pos, size_t *left, const char *fmt, ...) extern bool +is_tty(int fd) +{ +#if defined(_WIN32) && !defined(__CYGWIN__) + // There is no need to check if handle == INVALID_HANDLE_VALUE + // because it will return false anyway when used in GetConsoleMode(). + // The resulting HANDLE is owned by the file descriptor. + // The HANDLE must not be closed here. + intptr_t handle = _get_osfhandle(fd); + DWORD mode; + + // GetConsoleMode() is an easy way to tell if the HANDLE is a + // console or not. We do not care about the value of mode since we + // do not plan to use any further Windows console functions. + return GetConsoleMode((HANDLE)handle, &mode); +#else + return isatty(fd); +#endif +} + + +extern bool is_tty_stdin(void) { - const bool ret = isatty(STDIN_FILENO); + const bool ret = is_tty(STDIN_FILENO); if (ret) message_error(_("Compressed data cannot be read from " @@ -277,7 +297,7 @@ is_tty_stdin(void) extern bool is_tty_stdout(void) { - const bool ret = isatty(STDOUT_FILENO); + const bool ret = is_tty(STDOUT_FILENO); if (ret) message_error(_("Compressed data cannot be written to " diff --git a/src/xz/util.h b/src/xz/util.h index 6d7e148..a2fdd05 100644 --- a/src/xz/util.h +++ b/src/xz/util.h @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file util.h @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// /// \brief Safe malloc() that never returns NULL @@ -105,6 +104,20 @@ lzma_attribute((__format__(__printf__, 3, 4))) extern void my_snprintf(char **pos, size_t *left, const char *fmt, ...); +/// \brief Test if file descriptor is a terminal +/// +/// For POSIX systems, this is a simple wrapper around isatty(). However on +/// Windows, isatty() returns true for all character devices, not just +/// terminals. +/// +/// \param fd File descriptor to test +/// +/// \return bool: +/// - true if file descriptor is a terminal +/// - false otherwise +extern bool is_tty(int fd); + + /// \brief Test if stdin is a terminal /// /// If stdin is a terminal, an error message is printed and exit status set diff --git a/src/xz/xz.1 b/src/xz/xz.1 index 73ca6ef..5b880e8 100644 --- a/src/xz/xz.1 +++ b/src/xz/xz.1 @@ -1,12 +1,10 @@ '\" t +.\" SPDX-License-Identifier: 0BSD .\" .\" Authors: Lasse Collin .\" Jia Tan .\" -.\" This file has been put into the public domain. -.\" You can do whatever you want with this file. -.\" -.TH XZ 1 "2023-07-17" "Tukaani" "XZ Utils" +.TH XZ 1 "2024-04-08" "Tukaani" "XZ Utils" . .SH NAME xz, unxz, xzcat, lzma, unlzma, lzcat \- Compress or decompress .xz and .lzma files @@ -801,8 +799,6 @@ in the single-threaded mode. It may vary slightly between .B xz versions. -Memory requirements of some of the future multithreaded modes may -be dramatically higher than that of the single-threaded mode. .IP \(bu 3 DecMem contains the decompressor memory requirements. That is, the compression settings determine @@ -811,6 +807,15 @@ The exact decompressor memory usage is slightly more than the LZMA2 dictionary size, but the values in the table have been rounded up to the next full MiB. .RE +.IP "" +Memory requirements of the multi-threaded mode are +significantly higher than that of the single-threaded mode. +With the default value of +.BR \-\-block\-size , +each thread needs 3*3*DictSize plus CompMem or DecMem. +For example, four threads with preset +.B \-6 +needs 660\(en670\ MiB of memory. .TP .BR \-e ", " \-\-extreme Use a slower variant of the selected compression preset level @@ -902,50 +907,90 @@ Using .I size less than the LZMA2 dictionary size is waste of RAM because then the LZMA2 dictionary buffer will never get fully used. -The sizes of the blocks are stored in the block headers, -which a future version of -.B xz -will use for multi-threaded decompression. +In multi-threaded mode, +the sizes of the blocks are stored in the block headers. +This size information is required for multi-threaded decompression. .IP "" In single-threaded mode no block splitting is done by default. Setting this option doesn't affect memory usage. No size information is stored in block headers, thus files created in single-threaded mode won't be identical to files created in multi-threaded mode. -The lack of size information also means that a future version of +The lack of size information also means that .B xz won't be able decompress the files in multi-threaded mode. .TP -.BI \-\-block\-list= sizes +.BI \-\-block\-list= items When compressing to the .B .xz -format, start a new block after +format, start a new block with an optional custom filter chain after the given intervals of uncompressed data. .IP "" -The uncompressed -.I sizes -of the blocks are specified as a comma-separated list. -Omitting a size (two or more consecutive commas) is a shorthand -to use the size of the previous block. +The +.I items +are a comma-separated list. +Each item consists of an optional filter chain number +between 0 and 9 followed by a colon +.RB ( : ) +and a required size of uncompressed data. +Omitting an item (two or more consecutive commas) is a +shorthand to use the size and filters of the previous item. .IP "" If the input file is bigger than the sum of -.IR sizes , -the last value in -.I sizes -is repeated until the end of the file. +the sizes in +.IR items , +the last item is repeated until the end of the file. A special value of .B 0 -may be used as the last value to indicate that +may be used as the last size to indicate that the rest of the file should be encoded as a single block. .IP "" -If one specifies -.I sizes -that exceed the encoder's block size +An alternative filter chain for each block can be +specified in combination with the +.BI \-\-filters1= filters +\&...\& +.BI \-\-filters9= filters +options. +These options define filter chains with an identifier +between 1\(en9. +Filter chain 0 can be used to refer to the default filter chain, +which is the same as not specifying a filter chain. +The filter chain identifier can be used before the uncompressed +size, followed by a colon +.RB ( : ). +For example, if one specifies +.B \-\-block\-list=1:2MiB,3:2MiB,2:4MiB,,2MiB,0:4MiB +then blocks will be created using: +.RS +.IP \(bu 3 +The filter chain specified by +.B \-\-filters1 +and 2 MiB input +.IP \(bu 3 +The filter chain specified by +.B \-\-filters3 +and 2 MiB input +.IP \(bu 3 +The filter chain specified by +.B \-\-filters2 +and 4 MiB input +.IP \(bu 3 +The filter chain specified by +.B \-\-filters2 +and 4 MiB input +.IP \(bu 3 +The default filter chain and 2 MiB input +.IP \(bu 3 +The default filter chain and 4 MiB input for every block until +end of input. +.RE +.IP "" +If one specifies a size that exceeds the encoder's block size (either the default value in threaded mode or the value specified with \fB\-\-block\-size=\fIsize\fR), the encoder will create additional blocks while keeping the boundaries specified in -.IR sizes . +.IR items . For example, if one specifies .B \-\-block\-size=10MiB .B \-\-block\-list=5MiB,10MiB,8MiB,12MiB,24MiB @@ -1262,6 +1307,15 @@ meet this condition, but files compressed in single-threaded mode don't even if .BI \-\-block\-size= size has been used. +.IP "" +The default value for +.I threads +is +.BR 0 . +In +.B xz +5.4.x and older the default is +.BR 1 . . .SS "Custom compressor filter chains" A custom filter chain allows specifying @@ -1295,22 +1349,37 @@ in the chain. Depending on the filter, this limitation is either inherent to the filter design or exists to prevent security issues. .PP -A custom filter chain is specified by using one or more -filter options in the order they are wanted in the filter chain. -That is, the order of filter options is significant! +A custom filter chain can be specified in two different ways. +The options +.BI \-\-filters= filters +and +.BI \-\-filters1= filters +\&...\& +.BI \-\-filters9= filters +allow specifying an entire filter chain in one option using the +liblzma filter string syntax. +Alternatively, a filter chain can be specified by using one or more +individual filter options in the order they are wanted in the filter chain. +That is, the order of the individual filter options is significant! When decoding raw streams .RB ( \-\-format=raw ), -the filter chain is specified in the same order as +the filter chain must be specified in the same order as it was specified when compressing. -.PP -Filters take filter-specific +Any individual filter or preset options specified before the full +chain option +(\fB\-\-filters=\fIfilters\fR) +will be forgotten. +Individual filters specified after the full chain option will reset the +filter chain. +.PP +Both the full and individual filter options take filter-specific .I options as a comma-separated list. Extra commas in .I options are ignored. -Every option has a default value, so you need to -specify only those you want to change. +Every option has a default value, so +specify those you want to change. .PP To see the whole filter chain and .IR options , @@ -1321,6 +1390,45 @@ use twice). This works also for viewing the filter chain options used by presets. .TP +.BI \-\-filters= filters +Specify the full filter chain or a preset in a single option. +Each filter can be separated by spaces or two dashes +.RB ( \-\- ). +.I filters +may need to be quoted on the shell command line so it is +parsed as a single option. +To denote +.IR options , +use +.B : +or +.BR = . +A preset can be prefixed with a +.B \- +and followed with zero or more flags. +The only supported flag is +.B e +to apply the same options as +.BR \-\-extreme . +.TP +\fB\-\-filters1\fR=\fIfilters\fR ... \fB\-\-filters9\fR=\fIfilters +Specify up to nine additional filter chains that can be used with +.BR \-\-block\-list . +.IP "" +For example, when compressing an archive with executable files +followed by text files, the executable part could use a filter +chain with a BCJ filter and the text part only the LZMA2 filter. +.TP +.B \-\-filters-help +Display a help message describing how to specify presets and +custom filter chains in the +.B \-\-filters +and +.BI \-\-filters1= filters +\&...\& +.BI \-\-filters9= filters +options, and exit successfully. +.TP \fB\-\-lzma1\fR[\fB=\fIoptions\fR] .PD 0 .TP @@ -1704,6 +1812,8 @@ and \fB\-\-ia64\fR[\fB=\fIoptions\fR] .TP \fB\-\-sparc\fR[\fB=\fIoptions\fR] +.TP +\fB\-\-riscv\fR[\fB=\fIoptions\fR] .PD Add a branch/call/jump (BCJ) filter to the filter chain. These filters can be used only as a non-last filter @@ -1762,6 +1872,7 @@ ARM64;4;4096-byte alignment is best PowerPC;4;Big endian only IA-64;16;Itanium SPARC;4; +RISC-V;2; .TE .RE .RE @@ -1770,14 +1881,38 @@ Since the BCJ-filtered data is usually compressed with LZMA2, the compression ratio may be improved slightly if the LZMA2 options are set to match the alignment of the selected BCJ filter. -For example, with the IA-64 filter, it's good to set -.B pb=4 -or even +Examples: +.RS +.IP \(bu 3 +IA-64 filter has 16-byte alignment so .B pb=4,lp=4,lc=0 +is good with LZMA2 (2^4=16). -The x86 filter is an exception; -it's usually good to stick to LZMA2's default -four-byte alignment when compressing x86 executables. +.IP \(bu 3 +RISC-V code has 2-byte or 4-byte alignment +depending on whether the file contains +16-bit compressed instructions (the C extension). +When 16-bit instructions are used, +.B pb=2,lp=1,lc=3 +or +.B pb=1,lp=1,lc=3 +is good. +When 16-bit instructions aren't present, +.B pb=2,lp=2,lc=2 +is the best. +.B readelf \-h +can be used to check if "RVC" +appears on the "Flags" line. +.IP \(bu 3 +ARM64 is always 4-byte aligned so +.B pb=2,lp=2,lc=2 +is the best. +.IP \(bu 3 +The x86 filter is an exception. +It's usually good to stick to LZMA2's defaults +.RB ( pb=2,lp=0,lc=3 ) +when compressing x86 executables. +.RE .IP "" All BCJ filters support the same .IR options : @@ -1954,107 +2089,14 @@ easier to parse by other programs. Currently .B \-\-robot is supported only together with -.BR \-\-version , +.BR \-\-list , +.BR \-\-filters\-help , .BR \-\-info\-memory , and -.BR \-\-list . +.BR \-\-version . It will be supported for compression and decompression in the future. . -.SS Version -.B "xz \-\-robot \-\-version" -prints the version number of -.B xz -and liblzma in the following format: -.PP -.BI XZ_VERSION= XYYYZZZS -.br -.BI LIBLZMA_VERSION= XYYYZZZS -.TP -.I X -Major version. -.TP -.I YYY -Minor version. -Even numbers are stable. -Odd numbers are alpha or beta versions. -.TP -.I ZZZ -Patch level for stable releases or -just a counter for development releases. -.TP -.I S -Stability. -0 is alpha, 1 is beta, and 2 is stable. -.I S -should be always 2 when -.I YYY -is even. -.PP -.I XYYYZZZS -are the same on both lines if -.B xz -and liblzma are from the same XZ Utils release. -.PP -Examples: 4.999.9beta is -.B 49990091 -and -5.0.0 is -.BR 50000002 . -. -.SS "Memory limit information" -.B "xz \-\-robot \-\-info\-memory" -prints a single line with multiple tab-separated columns: -.IP 1. 4 -Total amount of physical memory (RAM) in bytes. -.IP 2. 4 -Memory usage limit for compression in bytes -.RB ( \-\-memlimit\-compress ). -A special value of -.B 0 -indicates the default setting -which for single-threaded mode is the same as no limit. -.IP 3. 4 -Memory usage limit for decompression in bytes -.RB ( \-\-memlimit\-decompress ). -A special value of -.B 0 -indicates the default setting -which for single-threaded mode is the same as no limit. -.IP 4. 4 -Since -.B xz -5.3.4alpha: -Memory usage for multi-threaded decompression in bytes -.RB ( \-\-memlimit\-mt\-decompress ). -This is never zero because a system-specific default value -shown in the column 5 -is used if no limit has been specified explicitly. -This is also never greater than the value in the column 3 -even if a larger value has been specified with -.BR \-\-memlimit\-mt\-decompress . -.IP 5. 4 -Since -.B xz -5.3.4alpha: -A system-specific default memory usage limit -that is used to limit the number of threads -when compressing with an automatic number of threads -.RB ( \-\-threads=0 ) -and no memory usage limit has been specified -.RB ( \-\-memlimit\-compress ). -This is also used as the default value for -.BR \-\-memlimit\-mt\-decompress . -.IP 6. 4 -Since -.B xz -5.3.4alpha: -Number of available processor threads. -.PP -In the future, the output of -.B "xz \-\-robot \-\-info\-memory" -may have more columns, but never more than a single line. -. .SS "List mode" .B "xz \-\-robot \-\-list" uses tab-separated output. @@ -2339,6 +2381,127 @@ Future versions may add new line types and new columns can be added to the existing line types, but the existing columns won't be changed. . +.SS "Filters help" +.B "xz \-\-robot \-\-filters-help" +prints the supported filters in the following format: +.PP +\fIfilter\fB:\fIoption\fB=<\fIvalue\fB>,\fIoption\fB=<\fIvalue\fB>\fR... +.TP +.I filter +Name of the filter +.TP +.I option +Name of a filter specific option +.TP +.I value +Numeric +.I value +ranges appear as +\fB<\fImin\fB\-\fImax\fB>\fR. +String +.I value +choices are shown within +.B "< >" +and separated by a +.B | +character. +.PP +Each filter is printed on its own line. +. +.SS "Memory limit information" +.B "xz \-\-robot \-\-info\-memory" +prints a single line with multiple tab-separated columns: +.IP 1. 4 +Total amount of physical memory (RAM) in bytes. +.IP 2. 4 +Memory usage limit for compression in bytes +.RB ( \-\-memlimit\-compress ). +A special value of +.B 0 +indicates the default setting +which for single-threaded mode is the same as no limit. +.IP 3. 4 +Memory usage limit for decompression in bytes +.RB ( \-\-memlimit\-decompress ). +A special value of +.B 0 +indicates the default setting +which for single-threaded mode is the same as no limit. +.IP 4. 4 +Since +.B xz +5.3.4alpha: +Memory usage for multi-threaded decompression in bytes +.RB ( \-\-memlimit\-mt\-decompress ). +This is never zero because a system-specific default value +shown in the column 5 +is used if no limit has been specified explicitly. +This is also never greater than the value in the column 3 +even if a larger value has been specified with +.BR \-\-memlimit\-mt\-decompress . +.IP 5. 4 +Since +.B xz +5.3.4alpha: +A system-specific default memory usage limit +that is used to limit the number of threads +when compressing with an automatic number of threads +.RB ( \-\-threads=0 ) +and no memory usage limit has been specified +.RB ( \-\-memlimit\-compress ). +This is also used as the default value for +.BR \-\-memlimit\-mt\-decompress . +.IP 6. 4 +Since +.B xz +5.3.4alpha: +Number of available processor threads. +.PP +In the future, the output of +.B "xz \-\-robot \-\-info\-memory" +may have more columns, but never more than a single line. +. +.SS Version +.B "xz \-\-robot \-\-version" +prints the version number of +.B xz +and liblzma in the following format: +.PP +.BI XZ_VERSION= XYYYZZZS +.br +.BI LIBLZMA_VERSION= XYYYZZZS +.TP +.I X +Major version. +.TP +.I YYY +Minor version. +Even numbers are stable. +Odd numbers are alpha or beta versions. +.TP +.I ZZZ +Patch level for stable releases or +just a counter for development releases. +.TP +.I S +Stability. +0 is alpha, 1 is beta, and 2 is stable. +.I S +should be always 2 when +.I YYY +is even. +.PP +.I XYYYZZZS +are the same on both lines if +.B xz +and liblzma are from the same XZ Utils release. +.PP +Examples: 4.999.9beta is +.B 49990091 +and +5.0.0 is +.BR 50000002 . +. .SH "EXIT STATUS" .TP .B 0 @@ -2391,7 +2554,7 @@ is run by a script or tool, for example, GNU .RS .PP .nf -.ft CW +.ft CR XZ_OPT=\-2v tar caf foo.tar.xz foo .ft R .fi @@ -2411,7 +2574,7 @@ scripts one may use something like this: .RS .PP .nf -.ft CW +.ft CR XZ_OPT=${XZ_OPT\-"\-7e"} export XZ_OPT .ft R @@ -2669,7 +2832,7 @@ if compression is successful: .RS .PP .nf -.ft CW +.ft CR xz foo .ft R .fi @@ -2685,7 +2848,7 @@ even if decompression is successful: .RS .PP .nf -.ft CW +.ft CR xz \-dk bar.xz .ft R .fi @@ -2703,7 +2866,7 @@ and 5\ MiB, respectively): .RS .PP .nf -.ft CW +.ft CR tar cf \- baz | xz \-4e > baz.tar.xz .ft R .fi @@ -2714,7 +2877,7 @@ to standard output with a single command: .RS .PP .nf -.ft CW +.ft CR xz \-dcf a.txt b.txt.xz c.txt d.txt.lzma > abcd.txt .ft R .fi @@ -2729,7 +2892,7 @@ can be used to parallelize compression of many files: .RS .PP .nf -.ft CW +.ft CR find . \-type f \e! \-name '*.xz' \-print0 \e | xargs \-0r \-P4 \-n16 xz \-T1 .ft R @@ -2769,7 +2932,7 @@ after compressing multiple files: .RS .PP .nf -.ft CW +.ft CR xz \-\-robot \-\-list *.xz | awk '/^totals/{print $5\-$4}' .ft R .fi @@ -2789,7 +2952,7 @@ option: .RS .PP .nf -.ft CW +.ft CR if ! eval "$(xz \-\-robot \-\-version 2> /dev/null)" || [ "$XZ_VERSION" \-lt 50000002 ]; then echo "Your xz is too old." @@ -2805,7 +2968,7 @@ but if a limit has already been set, don't increase it: .RS .PP .nf -.ft CW +.ft CR NEWLIM=$((123 << 20))\ \ # 123 MiB OLDLIM=$(xz \-\-robot \-\-info\-memory | cut \-f3) if [ $OLDLIM \-eq 0 \-o $OLDLIM \-gt $NEWLIM ]; then @@ -2858,7 +3021,7 @@ can be modified to use a bigger dictionary: .RS .PP .nf -.ft CW +.ft CR xz \-\-lzma2=preset=1,dict=32MiB foo.tar .ft R .fi @@ -2886,7 +3049,7 @@ would use: .RS .PP .nf -.ft CW +.ft CR xz \-vv \-\-lzma2=dict=192MiB big_foo.tar .ft R .fi @@ -2916,7 +3079,7 @@ using about 100\ KiB of memory. .RS .PP .nf -.ft CW +.ft CR xz \-\-check=crc32 \-\-lzma2=preset=6e,dict=64KiB foo .ft R .fi @@ -2944,7 +3107,7 @@ slightly (like 0.1\ %) smaller file than .RS .PP .nf -.ft CW +.ft CR xz \-\-lzma2=preset=6e,pb=0,lc=4 source_code.tar .ft R .fi @@ -2957,7 +3120,7 @@ using the x86 BCJ filter: .RS .PP .nf -.ft CW +.ft CR xz \-\-x86 \-\-lzma2 libfoo.so .ft R .fi @@ -2992,7 +3155,7 @@ to LZMA2 to accommodate the three-byte alignment: .RS .PP .nf -.ft CW +.ft CR xz \-\-delta=dist=3 \-\-lzma2=pb=0 foo.tiff .ft R .fi diff --git a/src/xz/xz_w32res.rc b/src/xz/xz_w32res.rc index bad3020..e80903e 100644 --- a/src/xz/xz_w32res.rc +++ b/src/xz/xz_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_APP diff --git a/src/xzdec/Makefile.am b/src/xzdec/Makefile.am index 90f1e92..31a6508 100644 --- a/src/xzdec/Makefile.am +++ b/src/xzdec/Makefile.am @@ -1,9 +1,5 @@ -## +## SPDX-License-Identifier: 0BSD ## Author: Lasse Collin -## -## This file has been put into the public domain. -## You can do whatever you want with this file. -## # Windows resource compiler support. It's fine to use xz_CPPFLAGS # also for lzmadec. diff --git a/src/xzdec/Makefile.in b/src/xzdec/Makefile.in index b05cff1..7d44f9b 100644 --- a/src/xzdec/Makefile.in +++ b/src/xzdec/Makefile.in @@ -103,8 +103,8 @@ bin_PROGRAMS = $(am__EXEEXT_1) $(am__EXEEXT_2) @COND_LZMADEC_TRUE@@COND_XZDEC_TRUE@am__append_6 = lzmadec subdir = src/xzdec ACLOCAL_M4 = $(top_srcdir)/aclocal.m4 -am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \ - $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \ +am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \ + $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \ $(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \ $(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \ $(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \ @@ -264,7 +264,6 @@ AUTOCONF = @AUTOCONF@ AUTOHEADER = @AUTOHEADER@ AUTOMAKE = @AUTOMAKE@ AWK = @AWK@ -CAPSICUM_LIB = @CAPSICUM_LIB@ CC = @CC@ CCAS = @CCAS@ CCASDEPMODE = @CCASDEPMODE@ @@ -393,6 +392,8 @@ install_sh = @install_sh@ libdir = @libdir@ libexecdir = @libexecdir@ localedir = @localedir@ +localedir_c = @localedir_c@ +localedir_c_make = @localedir_c_make@ localstatedir = @localstatedir@ mandir = @mandir@ mkdir_p = @mkdir_p@ diff --git a/src/xzdec/lzmadec_w32res.rc b/src/xzdec/lzmadec_w32res.rc index b85962d..b4b89b1 100644 --- a/src/xzdec/lzmadec_w32res.rc +++ b/src/xzdec/lzmadec_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_APP diff --git a/src/xzdec/xzdec.1 b/src/xzdec/xzdec.1 index 78bc9b4..5198efb 100644 --- a/src/xzdec/xzdec.1 +++ b/src/xzdec/xzdec.1 @@ -1,10 +1,8 @@ +.\" SPDX-License-Identifier: 0BSD .\" .\" Author: Lasse Collin .\" -.\" This file has been put into the public domain. -.\" You can do whatever you want with this file. -.\" -.TH XZDEC 1 "2017-04-19" "Tukaani" "XZ Utils" +.TH XZDEC 1 "2024-04-08" "Tukaani" "XZ Utils" .SH NAME xzdec, lzmadec \- Small .xz and .lzma decompressors .SH SYNOPSIS diff --git a/src/xzdec/xzdec.c b/src/xzdec/xzdec.c index 556c548..4d88174 100644 --- a/src/xzdec/xzdec.c +++ b/src/xzdec/xzdec.c @@ -1,3 +1,5 @@ +// SPDX-License-Identifier: 0BSD + /////////////////////////////////////////////////////////////////////////////// // /// \file xzdec.c @@ -5,9 +7,6 @@ // // Author: Lasse Collin // -// This file has been put into the public domain. -// You can do whatever you want with this file. -// /////////////////////////////////////////////////////////////////////////////// #include "sysdefs.h" @@ -16,7 +15,30 @@ #include <stdarg.h> #include <errno.h> #include <stdio.h> -#include <unistd.h> + +#ifndef _MSC_VER +# include <unistd.h> +#endif + +#ifdef HAVE_CAP_RIGHTS_LIMIT +# include <sys/capsicum.h> +#endif + +#ifdef HAVE_LINUX_LANDLOCK +# include <linux/landlock.h> +# include <sys/prctl.h> +# include <sys/syscall.h> +# ifdef LANDLOCK_ACCESS_NET_BIND_TCP +# define LANDLOCK_ABI_MAX 4 +# else +# define LANDLOCK_ABI_MAX 3 +# endif +#endif + +#if defined(HAVE_CAP_RIGHTS_LIMIT) || defined(HAVE_PLEDGE) \ + || defined(HAVE_LINUX_LANDLOCK) +# define ENABLE_SANDBOX 1 +#endif #include "getopt.h" #include "tuklib_progname.h" @@ -25,6 +47,10 @@ #ifdef TUKLIB_DOSLIKE # include <fcntl.h> # include <io.h> +# ifdef _MSC_VER +# define fileno _fileno +# define setmode _setmode +# endif #endif @@ -273,9 +299,123 @@ uncompress(lzma_stream *strm, FILE *file, const char *filename) } +#ifdef ENABLE_SANDBOX +static void +sandbox_enter(int src_fd) +{ +#if defined(HAVE_CAP_RIGHTS_LIMIT) + // Capsicum needs FreeBSD 10.2 or later. + cap_rights_t rights; + + if (cap_enter()) + goto error; + + if (cap_rights_limit(src_fd, cap_rights_init(&rights, CAP_READ))) + goto error; + + // If not reading from stdin, remove all capabilities from it. + if (src_fd != STDIN_FILENO && cap_rights_limit( + STDIN_FILENO, cap_rights_clear(&rights))) + goto error; + + if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights, + CAP_WRITE))) + goto error; + + if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights, + CAP_WRITE))) + goto error; + +#elif defined(HAVE_PLEDGE) + // pledge() was introduced in OpenBSD 5.9. + if (pledge("stdio", "")) + goto error; + + (void)src_fd; + +#elif defined(HAVE_LINUX_LANDLOCK) + int landlock_abi = syscall(SYS_landlock_create_ruleset, + (void *)NULL, 0, LANDLOCK_CREATE_RULESET_VERSION); + + if (landlock_abi > 0) { + if (landlock_abi > LANDLOCK_ABI_MAX) + landlock_abi = LANDLOCK_ABI_MAX; + + const struct landlock_ruleset_attr attr = { + .handled_access_fs = (1ULL + << (12 + my_min(3, landlock_abi))) - 1, +# if LANDLOCK_ABI_MAX >= 4 + .handled_access_net = landlock_abi < 4 ? 0 : + (LANDLOCK_ACCESS_NET_BIND_TCP + | LANDLOCK_ACCESS_NET_CONNECT_TCP), +# endif + }; + + const int ruleset_fd = syscall(SYS_landlock_create_ruleset, + &attr, sizeof(attr), 0U); + if (ruleset_fd < 0) + goto error; + + // All files we need should have already been opened. Thus, + // we don't need to add any rules using landlock_add_rule(2) + // before activating the sandbox. + if (syscall(SYS_landlock_restrict_self, ruleset_fd, 0U) != 0) + goto error; + } + + (void)src_fd; + +#else +# error ENABLE_SANDBOX is defined but no sandboxing method was found. +#endif + + return; + +error: +#ifdef HAVE_CAP_RIGHTS_LIMIT + // If a kernel is configured without capability mode support or + // used in an emulator that does not implement the capability + // system calls, then the Capsicum system calls will fail and set + // errno to ENOSYS. In that case xzdec will silently run without + // the sandbox. + if (errno == ENOSYS) + return; +#endif + + my_errorf("Failed to enable the sandbox"); + exit(EXIT_FAILURE); +} +#endif + + int main(int argc, char **argv) { +#ifdef HAVE_PLEDGE + // OpenBSD's pledge(2) sandbox. + // Initially enable the sandbox slightly more relaxed so that + // the process can still open files. This allows the sandbox to + // be enabled when parsing command line arguments and decompressing + // all files (the more strict sandbox only restricts the last file + // that is decompressed). + if (pledge("stdio rpath", "")) { + my_errorf("Failed to enable the sandbox"); + exit(EXIT_FAILURE); + } +#endif + +#ifdef HAVE_LINUX_LANDLOCK + // Prevent the process from gaining new privileges. This must be done + // before landlock_restrict_self(2) but since we will never need new + // privileges, this call can be done here already. + // + // This is supported since Linux 3.5. Ignore the return value to + // keep compatibility with old kernels. landlock_restrict_self(2) + // will fail if the no_new_privs attribute isn't set, thus if prctl() + // fails here the error will still be detected when it matters. + (void)prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0); +#endif + // Initialize progname which we will be used in error messages. tuklib_progname_init(argv); @@ -295,24 +435,42 @@ main(int argc, char **argv) if (optind == argc) { // No filenames given, decode from stdin. +#ifdef ENABLE_SANDBOX + sandbox_enter(STDIN_FILENO); +#endif uncompress(&strm, stdin, "(stdin)"); } else { // Loop through the filenames given on the command line. do { + FILE *src_file; + const char *src_name; + // "-" indicates stdin. if (strcmp(argv[optind], "-") == 0) { - uncompress(&strm, stdin, "(stdin)"); + src_file = stdin; + src_name = "(stdin)"; } else { - FILE *file = fopen(argv[optind], "rb"); - if (file == NULL) { - my_errorf("%s: %s", argv[optind], + src_name = argv[optind]; + src_file = fopen(src_name, "rb"); + if (src_file == NULL) { + my_errorf("%s: %s", src_name, strerror(errno)); exit(EXIT_FAILURE); } - - uncompress(&strm, file, argv[optind]); - fclose(file); } +#ifdef ENABLE_SANDBOX + // Enable the strict sandbox for the last file. + // Then the process can no longer open additional + // files. The typical xzdec use case is to decompress + // a single file so this way the strictest sandboxing + // is used in most cases. + if (optind == argc - 1) + sandbox_enter(fileno(src_file)); +#endif + uncompress(&strm, src_file, src_name); + + if (src_file != stdin) + (void)fclose(src_file); } while (++optind < argc); } diff --git a/src/xzdec/xzdec_w32res.rc b/src/xzdec/xzdec_w32res.rc index 0e26a22..fdbb102 100644 --- a/src/xzdec/xzdec_w32res.rc +++ b/src/xzdec/xzdec_w32res.rc @@ -1,8 +1,7 @@ +/* SPDX-License-Identifier: 0BSD */ + /* * Author: Lasse Collin - * - * This file has been put into the public domain. - * You can do whatever you want with this file. */ #define MY_TYPE VFT_APP |