summaryrefslogtreecommitdiffstats
path: root/src/xz
diff options
context:
space:
mode:
Diffstat (limited to 'src/xz')
-rw-r--r--src/xz/Makefile.am10
-rw-r--r--src/xz/Makefile.in57
-rw-r--r--src/xz/args.c186
-rw-r--r--src/xz/args.h8
-rw-r--r--src/xz/coder.c607
-rw-r--r--src/xz/coder.h42
-rw-r--r--src/xz/file_io.c144
-rw-r--r--src/xz/file_io.h21
-rw-r--r--src/xz/hardware.c11
-rw-r--r--src/xz/hardware.h5
-rw-r--r--src/xz/list.c31
-rw-r--r--src/xz/list.h5
-rw-r--r--src/xz/main.c91
-rw-r--r--src/xz/main.h5
-rw-r--r--src/xz/message.c94
-rw-r--r--src/xz/message.h16
-rw-r--r--src/xz/mytime.c105
-rw-r--r--src/xz/mytime.h11
-rw-r--r--src/xz/options.c7
-rw-r--r--src/xz/options.h5
-rw-r--r--src/xz/private.h26
-rw-r--r--src/xz/sandbox.c355
-rw-r--r--src/xz/sandbox.h43
-rw-r--r--src/xz/signals.c29
-rw-r--r--src/xz/signals.h5
-rw-r--r--src/xz/suffix.c17
-rw-r--r--src/xz/suffix.h5
-rw-r--r--src/xz/util.c36
-rw-r--r--src/xz/util.h19
-rw-r--r--src/xz/xz.1469
-rw-r--r--src/xz/xz_w32res.rc5
31 files changed, 1869 insertions, 601 deletions
diff --git a/src/xz/Makefile.am b/src/xz/Makefile.am
index 4bc64f3..4ff061f 100644
--- a/src/xz/Makefile.am
+++ b/src/xz/Makefile.am
@@ -1,9 +1,5 @@
-##
+## SPDX-License-Identifier: 0BSD
## Author: Lasse Collin
-##
-## This file has been put into the public domain.
-## You can do whatever you want with this file.
-##
bin_PROGRAMS = xz
@@ -25,6 +21,8 @@ xz_SOURCES = \
options.c \
options.h \
private.h \
+ sandbox.c \
+ sandbox.h \
signals.c \
signals.h \
suffix.c \
@@ -53,7 +51,7 @@ xz_CPPFLAGS = \
-I$(top_srcdir)/src/liblzma/api \
-I$(top_builddir)/lib
-xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(CAPSICUM_LIB)
+xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la
if COND_GNULIB
xz_LDADD += $(top_builddir)/lib/libgnu.a
diff --git a/src/xz/Makefile.in b/src/xz/Makefile.in
index e6422c5..18e0151 100644
--- a/src/xz/Makefile.in
+++ b/src/xz/Makefile.in
@@ -98,8 +98,8 @@ bin_PROGRAMS = xz$(EXEEXT)
@COND_LZMALINKS_TRUE@am__append_4 = lzma unlzma lzcat
subdir = src/xz
ACLOCAL_M4 = $(top_srcdir)/aclocal.m4
-am__aclocal_m4_deps = $(top_srcdir)/m4/ax_check_capsicum.m4 \
- $(top_srcdir)/m4/ax_pthread.m4 $(top_srcdir)/m4/getopt.m4 \
+am__aclocal_m4_deps = $(top_srcdir)/m4/ax_pthread.m4 \
+ $(top_srcdir)/m4/build-to-host.m4 $(top_srcdir)/m4/getopt.m4 \
$(top_srcdir)/m4/gettext.m4 $(top_srcdir)/m4/host-cpu-c-abi.m4 \
$(top_srcdir)/m4/iconv.m4 $(top_srcdir)/m4/intlmacosx.m4 \
$(top_srcdir)/m4/lib-ld.m4 $(top_srcdir)/m4/lib-link.m4 \
@@ -127,23 +127,25 @@ PROGRAMS = $(bin_PROGRAMS)
am__xz_SOURCES_DIST = args.c args.h coder.c coder.h file_io.c \
file_io.h hardware.c hardware.h main.c main.h message.c \
message.h mytime.c mytime.h options.c options.h private.h \
- signals.c signals.h suffix.c suffix.h util.c util.h \
- ../common/tuklib_open_stdxxx.c ../common/tuklib_progname.c \
- ../common/tuklib_exit.c ../common/tuklib_mbstr_width.c \
- ../common/tuklib_mbstr_fw.c list.c list.h xz_w32res.rc
+ sandbox.c sandbox.h signals.c signals.h suffix.c suffix.h \
+ util.c util.h ../common/tuklib_open_stdxxx.c \
+ ../common/tuklib_progname.c ../common/tuklib_exit.c \
+ ../common/tuklib_mbstr_width.c ../common/tuklib_mbstr_fw.c \
+ list.c list.h xz_w32res.rc
@COND_MAIN_DECODER_TRUE@am__objects_1 = xz-list.$(OBJEXT)
@COND_W32_TRUE@am__objects_2 = xz_w32res.$(OBJEXT)
am_xz_OBJECTS = xz-args.$(OBJEXT) xz-coder.$(OBJEXT) \
xz-file_io.$(OBJEXT) xz-hardware.$(OBJEXT) xz-main.$(OBJEXT) \
xz-message.$(OBJEXT) xz-mytime.$(OBJEXT) xz-options.$(OBJEXT) \
- xz-signals.$(OBJEXT) xz-suffix.$(OBJEXT) xz-util.$(OBJEXT) \
- xz-tuklib_open_stdxxx.$(OBJEXT) xz-tuklib_progname.$(OBJEXT) \
- xz-tuklib_exit.$(OBJEXT) xz-tuklib_mbstr_width.$(OBJEXT) \
- xz-tuklib_mbstr_fw.$(OBJEXT) $(am__objects_1) $(am__objects_2)
+ xz-sandbox.$(OBJEXT) xz-signals.$(OBJEXT) xz-suffix.$(OBJEXT) \
+ xz-util.$(OBJEXT) xz-tuklib_open_stdxxx.$(OBJEXT) \
+ xz-tuklib_progname.$(OBJEXT) xz-tuklib_exit.$(OBJEXT) \
+ xz-tuklib_mbstr_width.$(OBJEXT) xz-tuklib_mbstr_fw.$(OBJEXT) \
+ $(am__objects_1) $(am__objects_2)
xz_OBJECTS = $(am_xz_OBJECTS)
am__DEPENDENCIES_1 =
xz_DEPENDENCIES = $(top_builddir)/src/liblzma/liblzma.la \
- $(am__DEPENDENCIES_1) $(am__append_3) $(am__DEPENDENCIES_1)
+ $(am__append_3) $(am__DEPENDENCIES_1)
AM_V_lt = $(am__v_lt_@AM_V@)
am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@)
am__v_lt_0 = --silent
@@ -167,8 +169,9 @@ am__depfiles_remade = ./$(DEPDIR)/xz-args.Po ./$(DEPDIR)/xz-coder.Po \
./$(DEPDIR)/xz-file_io.Po ./$(DEPDIR)/xz-hardware.Po \
./$(DEPDIR)/xz-list.Po ./$(DEPDIR)/xz-main.Po \
./$(DEPDIR)/xz-message.Po ./$(DEPDIR)/xz-mytime.Po \
- ./$(DEPDIR)/xz-options.Po ./$(DEPDIR)/xz-signals.Po \
- ./$(DEPDIR)/xz-suffix.Po ./$(DEPDIR)/xz-tuklib_exit.Po \
+ ./$(DEPDIR)/xz-options.Po ./$(DEPDIR)/xz-sandbox.Po \
+ ./$(DEPDIR)/xz-signals.Po ./$(DEPDIR)/xz-suffix.Po \
+ ./$(DEPDIR)/xz-tuklib_exit.Po \
./$(DEPDIR)/xz-tuklib_mbstr_fw.Po \
./$(DEPDIR)/xz-tuklib_mbstr_width.Po \
./$(DEPDIR)/xz-tuklib_open_stdxxx.Po \
@@ -259,7 +262,6 @@ AUTOCONF = @AUTOCONF@
AUTOHEADER = @AUTOHEADER@
AUTOMAKE = @AUTOMAKE@
AWK = @AWK@
-CAPSICUM_LIB = @CAPSICUM_LIB@
CC = @CC@
CCAS = @CCAS@
CCASDEPMODE = @CCASDEPMODE@
@@ -388,6 +390,8 @@ install_sh = @install_sh@
libdir = @libdir@
libexecdir = @libexecdir@
localedir = @localedir@
+localedir_c = @localedir_c@
+localedir_c_make = @localedir_c_make@
localstatedir = @localstatedir@
mandir = @mandir@
mkdir_p = @mkdir_p@
@@ -408,8 +412,8 @@ top_srcdir = @top_srcdir@
xz = @xz@
xz_SOURCES = args.c args.h coder.c coder.h file_io.c file_io.h \
hardware.c hardware.h main.c main.h message.c message.h \
- mytime.c mytime.h options.c options.h private.h signals.c \
- signals.h suffix.c suffix.h util.c util.h \
+ mytime.c mytime.h options.c options.h private.h sandbox.c \
+ sandbox.h signals.c signals.h suffix.c suffix.h util.c util.h \
../common/tuklib_open_stdxxx.c ../common/tuklib_progname.c \
../common/tuklib_exit.c ../common/tuklib_mbstr_width.c \
../common/tuklib_mbstr_fw.c $(am__append_1) $(am__append_2)
@@ -421,8 +425,8 @@ xz_CPPFLAGS = \
# libgnu.a may need these libs, so this must be after libgnu.a.
-xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(CAPSICUM_LIB) \
- $(am__append_3) $(LTLIBINTL)
+xz_LDADD = $(top_builddir)/src/liblzma/liblzma.la $(am__append_3) \
+ $(LTLIBINTL)
dist_man_MANS = xz.1
xzlinks = unxz xzcat $(am__append_4)
all: all-am
@@ -527,6 +531,7 @@ distclean-compile:
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-message.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-mytime.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-options.Po@am__quote@ # am--include-marker
+@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-sandbox.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-signals.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-suffix.Po@am__quote@ # am--include-marker
@AMDEP_TRUE@@am__include@ @am__quote@./$(DEPDIR)/xz-tuklib_exit.Po@am__quote@ # am--include-marker
@@ -675,6 +680,20 @@ xz-options.obj: options.c
@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-options.obj `if test -f 'options.c'; then $(CYGPATH_W) 'options.c'; else $(CYGPATH_W) '$(srcdir)/options.c'; fi`
+xz-sandbox.o: sandbox.c
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-sandbox.o -MD -MP -MF $(DEPDIR)/xz-sandbox.Tpo -c -o xz-sandbox.o `test -f 'sandbox.c' || echo '$(srcdir)/'`sandbox.c
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-sandbox.Tpo $(DEPDIR)/xz-sandbox.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='sandbox.c' object='xz-sandbox.o' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-sandbox.o `test -f 'sandbox.c' || echo '$(srcdir)/'`sandbox.c
+
+xz-sandbox.obj: sandbox.c
+@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-sandbox.obj -MD -MP -MF $(DEPDIR)/xz-sandbox.Tpo -c -o xz-sandbox.obj `if test -f 'sandbox.c'; then $(CYGPATH_W) 'sandbox.c'; else $(CYGPATH_W) '$(srcdir)/sandbox.c'; fi`
+@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-sandbox.Tpo $(DEPDIR)/xz-sandbox.Po
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ $(AM_V_CC)source='sandbox.c' object='xz-sandbox.obj' libtool=no @AMDEPBACKSLASH@
+@AMDEP_TRUE@@am__fastdepCC_FALSE@ DEPDIR=$(DEPDIR) $(CCDEPMODE) $(depcomp) @AMDEPBACKSLASH@
+@am__fastdepCC_FALSE@ $(AM_V_CC@am__nodep@)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -c -o xz-sandbox.obj `if test -f 'sandbox.c'; then $(CYGPATH_W) 'sandbox.c'; else $(CYGPATH_W) '$(srcdir)/sandbox.c'; fi`
+
xz-signals.o: signals.c
@am__fastdepCC_TRUE@ $(AM_V_CC)$(CC) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(xz_CPPFLAGS) $(CPPFLAGS) $(AM_CFLAGS) $(CFLAGS) -MT xz-signals.o -MD -MP -MF $(DEPDIR)/xz-signals.Tpo -c -o xz-signals.o `test -f 'signals.c' || echo '$(srcdir)/'`signals.c
@am__fastdepCC_TRUE@ $(AM_V_at)$(am__mv) $(DEPDIR)/xz-signals.Tpo $(DEPDIR)/xz-signals.Po
@@ -985,6 +1004,7 @@ distclean: distclean-am
-rm -f ./$(DEPDIR)/xz-message.Po
-rm -f ./$(DEPDIR)/xz-mytime.Po
-rm -f ./$(DEPDIR)/xz-options.Po
+ -rm -f ./$(DEPDIR)/xz-sandbox.Po
-rm -f ./$(DEPDIR)/xz-signals.Po
-rm -f ./$(DEPDIR)/xz-suffix.Po
-rm -f ./$(DEPDIR)/xz-tuklib_exit.Po
@@ -1049,6 +1069,7 @@ maintainer-clean: maintainer-clean-am
-rm -f ./$(DEPDIR)/xz-message.Po
-rm -f ./$(DEPDIR)/xz-mytime.Po
-rm -f ./$(DEPDIR)/xz-options.Po
+ -rm -f ./$(DEPDIR)/xz-sandbox.Po
-rm -f ./$(DEPDIR)/xz-signals.Po
-rm -f ./$(DEPDIR)/xz-suffix.Po
-rm -f ./$(DEPDIR)/xz-tuklib_exit.Po
diff --git a/src/xz/args.c b/src/xz/args.c
index 17e778c..eba1b97 100644
--- a/src/xz/args.c
+++ b/src/xz/args.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file args.c
@@ -5,10 +7,8 @@
///
/// \note Filter-specific options parsing is in options.c.
//
-// Author: Lasse Collin
-//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
+// Authors: Lasse Collin
+// Jia Tan
//
///////////////////////////////////////////////////////////////////////////////
@@ -83,14 +83,20 @@ parse_block_list(const char *str_const)
++count;
// Prevent an unlikely integer overflow.
- if (count > SIZE_MAX / sizeof(uint64_t) - 1)
+ if (count > SIZE_MAX / sizeof(block_list_entry) - 1)
message_fatal(_("%s: Too many arguments to --block-list"),
str);
// Allocate memory to hold all the sizes specified.
// If --block-list was specified already, its value is forgotten.
free(opt_block_list);
- opt_block_list = xmalloc((count + 1) * sizeof(uint64_t));
+ opt_block_list = xmalloc((count + 1) * sizeof(block_list_entry));
+
+ // Clear the bitmask of filter chains in use.
+ block_list_chain_mask = 0;
+
+ // Reset the largest Block size found in --block-list.
+ block_list_largest = 0;
for (size_t i = 0; i < count; ++i) {
// Locate the next comma and replace it with \0.
@@ -98,6 +104,43 @@ parse_block_list(const char *str_const)
if (p != NULL)
*p = '\0';
+ // Use the default filter chain unless overridden.
+ opt_block_list[i].chain_num = 0;
+
+ // To specify a filter chain, the block list entry may be
+ // prepended with "[filter-chain-number]:". The size is
+ // still required for every block.
+ // For instance:
+ // --block-list=2:10MiB,1:5MiB,,8MiB,0:0
+ //
+ // Translates to:
+ // 1. Block of 10 MiB using filter chain 2
+ // 2. Block of 5 MiB using filter chain 1
+ // 3. Block of 5 MiB using filter chain 1
+ // 4. Block of 8 MiB using the default filter chain
+ // 5. The last block uses the default filter chain
+ //
+ // The block list:
+ // --block-list=2:MiB,1:,0
+ //
+ // Is not allowed because the second block does not specify
+ // the block size, only the filter chain.
+ if (str[0] >= '0' && str[0] <= '9' && str[1] == ':') {
+ if (str[2] == '\0')
+ message_fatal(_("In --block-list, block "
+ "size is missing after "
+ "filter chain number '%c:'"),
+ str[0]);
+
+ const unsigned chain_num = (unsigned)(str[0] - '0');
+ opt_block_list[i].chain_num = chain_num;
+ block_list_chain_mask |= 1U << chain_num;
+ str += 2;
+ } else {
+ // This Block uses the default filter chain.
+ block_list_chain_mask |= 1U << 0;
+ }
+
if (str[0] == '\0') {
// There is no string, that is, a comma follows
// another comma. Use the previous value.
@@ -107,25 +150,39 @@ parse_block_list(const char *str_const)
assert(i > 0);
opt_block_list[i] = opt_block_list[i - 1];
} else {
- opt_block_list[i] = str_to_uint64("block-list", str,
- 0, UINT64_MAX);
+ opt_block_list[i].size = str_to_uint64("block-list",
+ str, 0, UINT64_MAX);
// Zero indicates no more new Blocks.
- if (opt_block_list[i] == 0) {
+ if (opt_block_list[i].size == 0) {
if (i + 1 != count)
message_fatal(_("0 can only be used "
"as the last element "
"in --block-list"));
- opt_block_list[i] = UINT64_MAX;
+ opt_block_list[i].size = UINT64_MAX;
}
+
+ // Remember the largest Block size in the list.
+ //
+ // NOTE: Do this after handling the special value 0
+ // because when 0 is used, we don't want to reduce
+ // the Block size of the multithreaded encoder.
+ if (block_list_largest < opt_block_list[i].size)
+ block_list_largest = opt_block_list[i].size;
}
- str = p + 1;
+ // Be standards compliant: p + 1 is undefined behavior
+ // if p == NULL. That occurs on the last iteration of
+ // the loop when we won't care about the value of str
+ // anymore anyway. That is, this is done conditionally
+ // solely for standard conformance reasons.
+ if (p != NULL)
+ str = p + 1;
}
// Terminate the array.
- opt_block_list[count] = 0;
+ opt_block_list[count].size = 0;
free(str_start);
return;
@@ -136,13 +193,26 @@ static void
parse_real(args_info *args, int argc, char **argv)
{
enum {
- OPT_X86 = INT_MIN,
+ OPT_FILTERS = INT_MIN,
+ OPT_FILTERS1,
+ OPT_FILTERS2,
+ OPT_FILTERS3,
+ OPT_FILTERS4,
+ OPT_FILTERS5,
+ OPT_FILTERS6,
+ OPT_FILTERS7,
+ OPT_FILTERS8,
+ OPT_FILTERS9,
+ OPT_FILTERS_HELP,
+
+ OPT_X86,
OPT_POWERPC,
OPT_IA64,
OPT_ARM,
OPT_ARMTHUMB,
OPT_ARM64,
OPT_SPARC,
+ OPT_RISCV,
OPT_DELTA,
OPT_LZMA1,
OPT_LZMA2,
@@ -191,7 +261,7 @@ parse_real(args_info *args, int argc, char **argv)
{ "check", required_argument, NULL, 'C' },
{ "ignore-check", no_argument, NULL, OPT_IGNORE_CHECK },
{ "block-size", required_argument, NULL, OPT_BLOCK_SIZE },
- { "block-list", required_argument, NULL, OPT_BLOCK_LIST },
+ { "block-list", required_argument, NULL, OPT_BLOCK_LIST },
{ "memlimit-compress", required_argument, NULL, OPT_MEM_COMPRESS },
{ "memlimit-decompress", required_argument, NULL, OPT_MEM_DECOMPRESS },
{ "memlimit-mt-decompress", required_argument, NULL, OPT_MEM_MT_DECOMPRESS },
@@ -206,6 +276,18 @@ parse_real(args_info *args, int argc, char **argv)
{ "best", no_argument, NULL, '9' },
// Filters
+ { "filters", optional_argument, NULL, OPT_FILTERS},
+ { "filters1", optional_argument, NULL, OPT_FILTERS1},
+ { "filters2", optional_argument, NULL, OPT_FILTERS2},
+ { "filters3", optional_argument, NULL, OPT_FILTERS3},
+ { "filters4", optional_argument, NULL, OPT_FILTERS4},
+ { "filters5", optional_argument, NULL, OPT_FILTERS5},
+ { "filters6", optional_argument, NULL, OPT_FILTERS6},
+ { "filters7", optional_argument, NULL, OPT_FILTERS7},
+ { "filters8", optional_argument, NULL, OPT_FILTERS8},
+ { "filters9", optional_argument, NULL, OPT_FILTERS9},
+ { "filters-help", optional_argument, NULL, OPT_FILTERS_HELP},
+
{ "lzma1", optional_argument, NULL, OPT_LZMA1 },
{ "lzma2", optional_argument, NULL, OPT_LZMA2 },
{ "x86", optional_argument, NULL, OPT_X86 },
@@ -215,6 +297,7 @@ parse_real(args_info *args, int argc, char **argv)
{ "armthumb", optional_argument, NULL, OPT_ARMTHUMB },
{ "arm64", optional_argument, NULL, OPT_ARM64 },
{ "sparc", optional_argument, NULL, OPT_SPARC },
+ { "riscv", optional_argument, NULL, OPT_RISCV },
{ "delta", optional_argument, NULL, OPT_DELTA },
// Other options
@@ -372,7 +455,30 @@ parse_real(args_info *args, int argc, char **argv)
opt_mode = MODE_COMPRESS;
break;
- // Filter setup
+ // --filters
+ case OPT_FILTERS:
+ coder_add_filters_from_str(optarg);
+ break;
+
+ // --filters1...--filters9
+ case OPT_FILTERS1:
+ case OPT_FILTERS2:
+ case OPT_FILTERS3:
+ case OPT_FILTERS4:
+ case OPT_FILTERS5:
+ case OPT_FILTERS6:
+ case OPT_FILTERS7:
+ case OPT_FILTERS8:
+ case OPT_FILTERS9:
+ coder_add_block_filters(optarg,
+ (size_t)(c - OPT_FILTERS));
+ break;
+
+ // --filters-help
+ case OPT_FILTERS_HELP:
+ // This doesn't return.
+ message_filters_help();
+ break;
case OPT_X86:
coder_add_filter(LZMA_FILTER_X86,
@@ -409,6 +515,11 @@ parse_real(args_info *args, int argc, char **argv)
options_bcj(optarg));
break;
+ case OPT_RISCV:
+ coder_add_filter(LZMA_FILTER_RISCV,
+ options_bcj(optarg));
+ break;
+
case OPT_DELTA:
coder_add_filter(LZMA_FILTER_DELTA,
options_delta(optarg));
@@ -516,8 +627,8 @@ parse_real(args_info *args, int argc, char **argv)
case OPT_FILES0:
if (args->files_name != NULL)
message_fatal(_("Only one file can be "
- "specified with `--files' "
- "or `--files0'."));
+ "specified with '--files' "
+ "or '--files0'."));
if (optarg == NULL) {
args->files_name = stdin_filename;
@@ -718,6 +829,39 @@ args_parse(args_info *args, int argc, char **argv)
if (opt_mode == MODE_COMPRESS && opt_format == FORMAT_AUTO)
opt_format = FORMAT_XZ;
+ // Set opt_block_list to NULL if we are not compressing to the .xz
+ // format. This option cannot be used outside of this case, and
+ // simplifies the implementation later.
+ if ((opt_mode != MODE_COMPRESS || opt_format != FORMAT_XZ)
+ && opt_block_list != NULL) {
+ message(V_WARNING, _("--block-list is ignored unless "
+ "compressing to the .xz format"));
+ free(opt_block_list);
+ opt_block_list = NULL;
+ }
+
+ // If raw format is used and a custom suffix is not provided,
+ // then only stdout mode can be used when compressing or
+ // decompressing.
+ if (opt_format == FORMAT_RAW && !suffix_is_set() && !opt_stdout
+ && (opt_mode == MODE_COMPRESS
+ || opt_mode == MODE_DECOMPRESS)) {
+ if (args->files_name != NULL)
+ message_fatal(_("With --format=raw, "
+ "--suffix=.SUF is required "
+ "unless writing to stdout"));
+
+ // If all of the filenames provided are "-" (more than one
+ // "-" could be specified) or no filenames are provided,
+ // then we are only going to be writing to standard out.
+ for (int i = optind; i < argc; i++) {
+ if (strcmp(argv[i], "-") != 0)
+ message_fatal(_("With --format=raw, "
+ "--suffix=.SUF is required "
+ "unless writing to stdout"));
+ }
+ }
+
// Compression settings need to be validated (options themselves and
// their memory usage) when compressing to any file format. It has to
// be done also when uncompressing raw data, since for raw decoding
@@ -727,14 +871,6 @@ args_parse(args_info *args, int argc, char **argv)
&& opt_mode != MODE_LIST))
coder_set_compression_settings();
- // If raw format is used and a custom suffix is not provided,
- // then only stdout mode can be used when compressing or decompressing.
- if (opt_format == FORMAT_RAW && !suffix_is_set() && !opt_stdout
- && (opt_mode == MODE_COMPRESS
- || opt_mode == MODE_DECOMPRESS))
- message_fatal(_("With --format=raw, --suffix=.SUF is "
- "required unless writing to stdout"));
-
// If no filenames are given, use stdin.
if (argv[optind] == NULL && args->files_name == NULL) {
// We don't modify or free() the "-" constant. The caller
diff --git a/src/xz/args.h b/src/xz/args.h
index a1a5930..e693ecd 100644
--- a/src/xz/args.h
+++ b/src/xz/args.h
@@ -1,12 +1,12 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file args.h
/// \brief Argument parsing
//
-// Author: Lasse Collin
-//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
+// Authors: Lasse Collin
+// Jia Tan
//
///////////////////////////////////////////////////////////////////////////////
diff --git a/src/xz/coder.c b/src/xz/coder.c
index 91d40ed..5e41f0d 100644
--- a/src/xz/coder.c
+++ b/src/xz/coder.c
@@ -1,16 +1,17 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file coder.c
/// \brief Compresses or uncompresses a file
//
-// Author: Lasse Collin
-//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
+// Authors: Lasse Collin
+// Jia Tan
//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
+#include "tuklib_integer.h"
/// Return value type for coder_init().
@@ -26,25 +27,47 @@ enum format_type opt_format = FORMAT_AUTO;
bool opt_auto_adjust = true;
bool opt_single_stream = false;
uint64_t opt_block_size = 0;
-uint64_t *opt_block_list = NULL;
-
+block_list_entry *opt_block_list = NULL;
+uint64_t block_list_largest;
+uint32_t block_list_chain_mask;
/// Stream used to communicate with liblzma
static lzma_stream strm = LZMA_STREAM_INIT;
-/// Filters needed for all encoding all formats, and also decoding in raw data
-static lzma_filter filters[LZMA_FILTERS_MAX + 1];
+/// Maximum number of filter chains. The first filter chain is the default,
+/// and 9 other filter chains can be specified with --filtersX.
+#define NUM_FILTER_CHAIN_MAX 10
+
+/// The default filter chain is in chains[0]. It is used for encoding
+/// in all supported formats and also for decdoing raw streams. The other
+/// filter chains are set by --filtersX to support changing filters with
+/// the --block-list option.
+static lzma_filter chains[NUM_FILTER_CHAIN_MAX][LZMA_FILTERS_MAX + 1];
+
+/// Bitmask indicating which filter chains are actually used when encoding
+/// in the .xz format. This is needed since the filter chains specified using
+/// --filtersX (or the default filter chain) might in reality be unneeded
+/// if they are never used in --block-list. When --block-list isn't
+/// specified, only the default filter chain is used, thus the initial
+/// value of this variable is 1U << 0 (the number of the default chain is 0).
+static uint32_t chains_used_mask = 1U << 0;
/// Input and output buffers
static io_buf in_buf;
static io_buf out_buf;
-/// Number of filters. Zero indicates that we are using a preset.
+/// Number of filters in the default filter chain. Zero indicates that
+/// we are using a preset.
static uint32_t filters_count = 0;
/// Number of the preset (0-9)
static uint32_t preset_number = LZMA_PRESET_DEFAULT;
+/// True if the current default filter chain was set using the --filters
+/// option. The filter chain is reset if a preset option (like -9) or an
+/// old-style filter option (like --lzma2) is used after a --filters option.
+static bool string_to_filter_used = false;
+
/// Integrity check type
static lzma_check check;
@@ -60,7 +83,6 @@ static bool allow_trailing_input;
static lzma_mt mt_options = {
.flags = 0,
.timeout = 300,
- .filters = filters,
};
#endif
@@ -77,14 +99,14 @@ coder_set_check(lzma_check new_check)
static void
forget_filter_chain(void)
{
- // Setting a preset makes us forget a possibly defined custom
- // filter chain.
- while (filters_count > 0) {
- --filters_count;
- free(filters[filters_count].options);
- filters[filters_count].options = NULL;
+ // Setting a preset or using --filters makes us forget
+ // the earlier custom filter chain (if any).
+ if (filters_count > 0) {
+ lzma_filters_free(chains[0], NULL);
+ filters_count = 0;
}
+ string_to_filter_used = false;
return;
}
@@ -114,9 +136,15 @@ coder_add_filter(lzma_vli id, void *options)
if (filters_count == LZMA_FILTERS_MAX)
message_fatal(_("Maximum number of filters is four"));
- filters[filters_count].id = id;
- filters[filters_count].options = options;
- ++filters_count;
+ if (string_to_filter_used)
+ forget_filter_chain();
+
+ chains[0][filters_count].id = id;
+ chains[0][filters_count].options = options;
+
+ // Terminate the filter chain with LZMA_VLI_UNKNOWN to simplify
+ // implementation of forget_filter_chain().
+ chains[0][++filters_count].id = LZMA_VLI_UNKNOWN;
// Setting a custom filter chain makes us forget the preset options.
// This makes a difference if one specifies e.g. "xz -9 --lzma2 -e"
@@ -128,6 +156,69 @@ coder_add_filter(lzma_vli id, void *options)
}
+static void
+str_to_filters(const char *str, uint32_t index, uint32_t flags)
+{
+ int error_pos;
+ const char *err = lzma_str_to_filters(str, &error_pos,
+ chains[index], flags, NULL);
+
+ if (err != NULL) {
+ char filter_num[2] = "";
+ if (index > 0)
+ filter_num[0] = '0' + index;
+
+ // FIXME? The message in err isn't translated.
+ // Including the translations in the xz translations is
+ // slightly ugly but possible. Creating a new domain for
+ // liblzma might not be worth it especially since on some
+ // OSes it adds extra dependencies to translation libraries.
+ message(V_ERROR, _("Error in --filters%s=FILTERS option:"),
+ filter_num);
+ message(V_ERROR, "%s", str);
+ message(V_ERROR, "%*s^", error_pos, "");
+ message_fatal("%s", err);
+ }
+}
+
+
+extern void
+coder_add_filters_from_str(const char *filter_str)
+{
+ // Forget presets and previously defined filter chain. See
+ // coder_add_filter() above for why preset_number must be reset too.
+ forget_filter_chain();
+ preset_number = LZMA_PRESET_DEFAULT;
+
+ string_to_filter_used = true;
+
+ // Include LZMA_STR_ALL_FILTERS so this can be used with --format=raw.
+ str_to_filters(filter_str, 0, LZMA_STR_ALL_FILTERS);
+
+ // Set the filters_count to be the number of filters converted from
+ // the string.
+ for (filters_count = 0; chains[0][filters_count].id
+ != LZMA_VLI_UNKNOWN;
+ ++filters_count) ;
+
+ assert(filters_count > 0);
+ return;
+}
+
+
+extern void
+coder_add_block_filters(const char *str, size_t slot)
+{
+ // Free old filters first, if they were previously allocated.
+ if (chains_used_mask & (1U << slot))
+ lzma_filters_free(chains[slot], NULL);
+
+ str_to_filters(str, slot, 0);
+
+ chains_used_mask |= 1U << slot;
+}
+
+
tuklib_attr_noreturn
static void
memlimit_too_small(uint64_t memory_usage)
@@ -139,6 +230,67 @@ memlimit_too_small(uint64_t memory_usage)
}
+#ifdef HAVE_ENCODERS
+/// \brief Calculate the memory usage of each filter chain.
+///
+/// \param chains_memusages If non-NULL, the memusage of the encoder
+/// or decoder for each chain is stored in
+/// this array.
+/// \param mt If non-NULL, calculate memory usage of
+/// multithreaded encoder.
+/// \param encode Whether to calculate encoder or decoder
+/// memory usage. This must be true if
+/// mt != NULL.
+///
+/// \return Return the highest memory usage of all of the filter chains.
+static uint64_t
+get_chains_memusage(uint64_t *chains_memusages, const lzma_mt *mt, bool encode)
+{
+ uint64_t max_memusage = 0;
+
+#ifdef MYTHREAD_ENABLED
+ // Copy multithreading options to a temporary struct since the
+ // "filters" member needs to be changed.
+ lzma_mt mt_local;
+ if (mt != NULL)
+ mt_local = *mt;
+#else
+ (void)mt;
+#endif
+
+ for (uint32_t i = 0; i < ARRAY_SIZE(chains); i++) {
+ if (!(chains_used_mask & (1U << i)))
+ continue;
+
+ uint64_t memusage = UINT64_MAX;
+#ifdef MYTHREAD_ENABLED
+ if (mt != NULL) {
+ assert(encode);
+ mt_local.filters = chains[i];
+ memusage = lzma_stream_encoder_mt_memusage(&mt_local);
+ } else
+#endif
+ if (encode) {
+ memusage = lzma_raw_encoder_memusage(chains[i]);
+ }
+#ifdef HAVE_DECODERS
+ else {
+ memusage = lzma_raw_decoder_memusage(chains[i]);
+ }
+#endif
+
+ if (chains_memusages != NULL)
+ chains_memusages[i] = memusage;
+
+ if (memusage > max_memusage)
+ max_memusage = memusage;
+ }
+
+ return max_memusage;
+}
+#endif
+
+
extern void
coder_set_compression_settings(void)
{
@@ -156,10 +308,56 @@ coder_set_compression_settings(void)
check = LZMA_CHECK_CRC32;
}
+#ifdef HAVE_ENCODERS
+ if (opt_block_list != NULL) {
+ // args.c ensures these.
+ assert(opt_mode == MODE_COMPRESS);
+ assert(opt_format == FORMAT_XZ);
+
+ // Find out if block_list_chain_mask has a bit set that
+ // isn't set in chains_used_mask.
+ const uint32_t missing_chains_mask
+ = (block_list_chain_mask ^ chains_used_mask)
+ & block_list_chain_mask;
+
+ // If a filter chain was specified in --block-list but no
+ // matching --filtersX option was used, exit with an error.
+ if (missing_chains_mask != 0) {
+ // Get the number of the first missing filter chain
+ // and show it in the error message.
+ const unsigned first_missing
+ = (unsigned)ctz32(missing_chains_mask);
+
+ message_fatal(_("filter chain %u used by "
+ "--block-list but not specified "
+ "with --filters%u="),
+ first_missing, first_missing);
+ }
+
+ // Omit the unused filter chains from mask of used chains.
+ //
+ // (FIXME? When built with debugging, coder_free() will free()
+ // the filter chains (except the default chain) which makes
+ // Valgrind show fewer reachable allocations. But coder_free()
+ // uses this mask to determine which chains to free. Thus it
+ // won't free the ones that are cleared here from the mask.
+ // In practice this doesn't matter.)
+ chains_used_mask &= block_list_chain_mask;
+ } else {
+ // Reset filters used mask in case --block-list is not
+ // used, but --filtersX is used.
+ chains_used_mask = 1U << 0;
+ }
+#endif
+
// Options for LZMA1 or LZMA2 in case we are using a preset.
static lzma_options_lzma opt_lzma;
- if (filters_count == 0) {
+ // The first filter in the chains[] array is for the default
+ // filter chain.
+ lzma_filter *default_filters = chains[0];
+
+ if (filters_count == 0 && chains_used_mask & 1) {
// We are using a preset. This is not a good idea in raw mode
// except when playing around with things. Different versions
// of this software may use different options in presets, and
@@ -179,46 +377,61 @@ coder_set_compression_settings(void)
message_bug();
// Use LZMA2 except with --format=lzma we use LZMA1.
- filters[0].id = opt_format == FORMAT_LZMA
+ default_filters[0].id = opt_format == FORMAT_LZMA
? LZMA_FILTER_LZMA1 : LZMA_FILTER_LZMA2;
- filters[0].options = &opt_lzma;
+ default_filters[0].options = &opt_lzma;
+
filters_count = 1;
- }
- // Terminate the filter options array.
- filters[filters_count].id = LZMA_VLI_UNKNOWN;
+ // Terminate the filter options array.
+ default_filters[1].id = LZMA_VLI_UNKNOWN;
+ }
// If we are using the .lzma format, allow exactly one filter
- // which has to be LZMA1.
+ // which has to be LZMA1. There is no need to check if the default
+ // filter chain is being used since it can only be disabled if
+ // --block-list is used, which is incompatible with FORMAT_LZMA.
if (opt_format == FORMAT_LZMA && (filters_count != 1
- || filters[0].id != LZMA_FILTER_LZMA1))
+ || default_filters[0].id != LZMA_FILTER_LZMA1))
message_fatal(_("The .lzma format supports only "
"the LZMA1 filter"));
// If we are using the .xz format, make sure that there is no LZMA1
- // filter to prevent LZMA_PROG_ERROR.
- if (opt_format == FORMAT_XZ)
+ // filter to prevent LZMA_PROG_ERROR. With the chains from --filtersX
+ // we have already ensured this by calling lzma_str_to_filters()
+ // without setting the flags that would allow non-.xz filters.
+ if (opt_format == FORMAT_XZ && chains_used_mask & 1)
for (size_t i = 0; i < filters_count; ++i)
- if (filters[i].id == LZMA_FILTER_LZMA1)
+ if (default_filters[i].id == LZMA_FILTER_LZMA1)
message_fatal(_("LZMA1 cannot be used "
"with the .xz format"));
- // Print the selected filter chain.
- message_filters_show(V_DEBUG, filters);
+ if (chains_used_mask & 1) {
+ // Print the selected default filter chain.
+ message_filters_show(V_DEBUG, default_filters);
+ }
// The --flush-timeout option requires LZMA_SYNC_FLUSH support
- // from the filter chain. Currently threaded encoder doesn't support
- // LZMA_SYNC_FLUSH so single-threaded mode must be used.
+ // from the filter chain. Currently the threaded encoder doesn't
+ // support LZMA_SYNC_FLUSH so single-threaded mode must be used.
if (opt_mode == MODE_COMPRESS && opt_flush_timeout != 0) {
- for (size_t i = 0; i < filters_count; ++i) {
- switch (filters[i].id) {
- case LZMA_FILTER_LZMA2:
- case LZMA_FILTER_DELTA:
- break;
+ for (unsigned i = 0; i < ARRAY_SIZE(chains); ++i) {
+ if (!(chains_used_mask & (1U << i)))
+ continue;
+
+ const lzma_filter *fc = chains[i];
+ for (size_t j = 0; fc[j].id != LZMA_VLI_UNKNOWN; j++) {
+ switch (fc[j].id) {
+ case LZMA_FILTER_LZMA2:
+ case LZMA_FILTER_DELTA:
+ break;
- default:
- message_fatal(_("The filter chain is "
- "incompatible with --flush-timeout"));
+ default:
+ message_fatal(_("Filter chain %u is "
+ "incompatible with "
+ "--flush-timeout"),
+ i);
+ }
}
}
@@ -229,23 +442,72 @@ coder_set_compression_settings(void)
}
}
- // Get the memory usage. Note that if --format=raw was used,
- // we can be decompressing.
+ // Get memory limit and the memory usage of the used filter chains.
+ // Note that if --format=raw was used, we can be decompressing
+ // using the default filter chain.
//
- // If multithreaded .xz compression is done, this value will be
- // replaced.
+ // If multithreaded .xz compression is done, the memory limit
+ // will be replaced.
uint64_t memory_limit = hardware_memlimit_get(opt_mode);
uint64_t memory_usage = UINT64_MAX;
+
+#ifdef HAVE_ENCODERS
+ // Memory usage for each encoder filter chain (default
+ // or --filtersX). The encoder options may need to be
+ // scaled down depending on the memory usage limit.
+ uint64_t encoder_memusages[ARRAY_SIZE(chains)];
+#endif
+
if (opt_mode == MODE_COMPRESS) {
#ifdef HAVE_ENCODERS
# ifdef MYTHREAD_ENABLED
if (opt_format == FORMAT_XZ && hardware_threads_is_mt()) {
memory_limit = hardware_memlimit_mtenc_get();
mt_options.threads = hardware_threads_get();
- mt_options.block_size = opt_block_size;
+
+ uint64_t block_size = opt_block_size;
+
+ // If opt_block_size is not set, find the maximum
+ // recommended Block size based on the filter chains
+ if (block_size == 0) {
+ for (unsigned i = 0; i < ARRAY_SIZE(chains);
+ i++) {
+ if (!(chains_used_mask & (1U << i)))
+ continue;
+
+ uint64_t size = lzma_mt_block_size(
+ chains[i]);
+
+ // If this returns an error, then one
+ // of the filter chains in use is
+ // invalid, so there is no point in
+ // progressing further.
+ if (size == UINT64_MAX)
+ message_fatal(_("Unsupported "
+ "options in filter "
+ "chain %u"), i);
+
+ if (size > block_size)
+ block_size = size;
+ }
+
+ // If --block-list was used and our current
+ // Block size exceeds the largest size
+ // in --block-list, reduce the Block size of
+ // the multithreaded encoder. The extra size
+ // would only be a waste of RAM. With a
+ // smaller Block size we might even be able
+ // to use more threads in some cases.
+ if (block_list_largest > 0 && block_size
+ > block_list_largest)
+ block_size = block_list_largest;
+ }
+
+ mt_options.block_size = block_size;
mt_options.check = check;
- memory_usage = lzma_stream_encoder_mt_memusage(
- &mt_options);
+
+ memory_usage = get_chains_memusage(encoder_memusages,
+ &mt_options, true);
if (memory_usage != UINT64_MAX)
message(V_DEBUG, _("Using up to %" PRIu32
" threads."),
@@ -253,12 +515,13 @@ coder_set_compression_settings(void)
} else
# endif
{
- memory_usage = lzma_raw_encoder_memusage(filters);
+ memory_usage = get_chains_memusage(encoder_memusages,
+ NULL, true);
}
#endif
} else {
#ifdef HAVE_DECODERS
- memory_usage = lzma_raw_decoder_memusage(filters);
+ memory_usage = lzma_raw_decoder_memusage(default_filters);
#endif
}
@@ -268,12 +531,13 @@ coder_set_compression_settings(void)
// Print memory usage info before possible dictionary
// size auto-adjusting.
//
- // NOTE: If only encoder support was built, we cannot show the
+ // NOTE: If only encoder support was built, we cannot show
// what the decoder memory usage will be.
message_mem_needed(V_DEBUG, memory_usage);
-#ifdef HAVE_DECODERS
- if (opt_mode == MODE_COMPRESS) {
- const uint64_t decmem = lzma_raw_decoder_memusage(filters);
+
+#if defined(HAVE_ENCODERS) && defined(HAVE_DECODERS)
+ if (opt_mode == MODE_COMPRESS && message_verbosity_get() >= V_DEBUG) {
+ const uint64_t decmem = get_chains_memusage(NULL, NULL, false);
if (decmem != UINT64_MAX)
message(V_DEBUG, _("Decompression will need "
"%s MiB of memory."), uint64_to_str(
@@ -300,14 +564,21 @@ coder_set_compression_settings(void)
// Reduce the number of threads by one and check
// the memory usage.
--mt_options.threads;
- memory_usage = lzma_stream_encoder_mt_memusage(
- &mt_options);
+ memory_usage = get_chains_memusage(encoder_memusages,
+ &mt_options, true);
if (memory_usage == UINT64_MAX)
message_bug();
if (memory_usage <= memory_limit) {
// The memory usage is now low enough.
- message(V_WARNING, _("Reduced the number of "
+ //
+ // Since 5.6.1: This is only shown at
+ // V_DEBUG instead of V_WARNING because
+ // changing the number of threads doesn't
+ // affect the output. On some systems this
+ // message would be too common now that
+ // multithreaded compression is the default.
+ message(V_DEBUG, _("Reduced the number of "
"threads from %s to %s to not exceed "
"the memory usage limit of %s MiB"),
uint64_to_str(
@@ -326,8 +597,11 @@ coder_set_compression_settings(void)
// way -T0 won't use insane amount of memory but at the same
// time the soft limit will never make xz fail and never make
// xz change settings that would affect the compressed output.
+ //
+ // Since 5.6.1: Like above, this is now shown at V_DEBUG
+ // instead of V_WARNING.
if (hardware_memlimit_mtenc_is_default()) {
- message(V_WARNING, _("Reduced the number of threads "
+ message(V_DEBUG, _("Reduced the number of threads "
"from %s to one. The automatic memory usage "
"limit of %s MiB is still being exceeded. "
"%s MiB of memory is required. "
@@ -353,7 +627,8 @@ coder_set_compression_settings(void)
// the multithreaded mode but the output
// is also different.
hardware_threads_set(1);
- memory_usage = lzma_raw_encoder_memusage(filters);
+ memory_usage = get_chains_memusage(encoder_memusages,
+ NULL, true);
message(V_WARNING, _("Switching to single-threaded mode "
"to not exceed the memory usage limit of %s MiB"),
uint64_to_str(round_up_to_mib(memory_limit), 0));
@@ -368,55 +643,85 @@ coder_set_compression_settings(void)
if (!opt_auto_adjust)
memlimit_too_small(memory_usage);
- // Look for the last filter if it is LZMA2 or LZMA1, so we can make
- // it use less RAM. With other filters we don't know what to do.
- size_t i = 0;
- while (filters[i].id != LZMA_FILTER_LZMA2
- && filters[i].id != LZMA_FILTER_LZMA1) {
- if (filters[i].id == LZMA_VLI_UNKNOWN)
- memlimit_too_small(memory_usage);
+ // Adjust each filter chain that is exceeding the memory usage limit.
+ for (unsigned i = 0; i < ARRAY_SIZE(chains); i++) {
+ // Skip unused chains.
+ if (!(chains_used_mask & (1U << i)))
+ continue;
+
+ // Skip chains that already meet the memory usage limit.
+ if (encoder_memusages[i] <= memory_limit)
+ continue;
+
+ // Look for the last filter if it is LZMA2 or LZMA1, so we
+ // can make it use less RAM. We cannot adjust other filters.
+ unsigned j = 0;
+ while (chains[i][j].id != LZMA_FILTER_LZMA2
+ && chains[i][j].id != LZMA_FILTER_LZMA1) {
+ // NOTE: This displays the too high limit of this
+ // particular filter chain. If multiple chains are
+ // specified and another one would need more then
+ // this message could be confusing. As long as LZMA2
+ // is the only memory hungry filter in .xz this
+ // doesn't matter at all in practice.
+ //
+ // FIXME? However, it's sort of odd still if we had
+ // switched from multithreaded mode to single-threaded
+ // mode because single-threaded produces different
+ // output. So the messages could perhaps be clearer.
+ // Another case of this is a few lines below.
+ if (chains[i][j].id == LZMA_VLI_UNKNOWN)
+ memlimit_too_small(encoder_memusages[i]);
+
+ ++j;
+ }
- ++i;
- }
+ // Decrease the dictionary size until we meet the memory
+ // usage limit. First round down to full mebibytes.
+ lzma_options_lzma *opt = chains[i][j].options;
+ const uint32_t orig_dict_size = opt->dict_size;
+ opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
- // Decrease the dictionary size until we meet the memory
- // usage limit. First round down to full mebibytes.
- lzma_options_lzma *opt = filters[i].options;
- const uint32_t orig_dict_size = opt->dict_size;
- opt->dict_size &= ~((UINT32_C(1) << 20) - 1);
- while (true) {
- // If it is below 1 MiB, auto-adjusting failed. We could be
- // more sophisticated and scale it down even more, but let's
- // see if many complain about this version.
- //
- // FIXME: Displays the scaled memory usage instead
- // of the original.
- if (opt->dict_size < (UINT32_C(1) << 20))
- memlimit_too_small(memory_usage);
+ while (true) {
+ // If it is below 1 MiB, auto-adjusting failed.
+ //
+ // FIXME? See the FIXME a few lines above.
+ if (opt->dict_size < (UINT32_C(1) << 20))
+ memlimit_too_small(encoder_memusages[i]);
- memory_usage = lzma_raw_encoder_memusage(filters);
- if (memory_usage == UINT64_MAX)
- message_bug();
+ encoder_memusages[i]
+ = lzma_raw_encoder_memusage(chains[i]);
+ if (encoder_memusages[i] == UINT64_MAX)
+ message_bug();
- // Accept it if it is low enough.
- if (memory_usage <= memory_limit)
- break;
+ // Accept it if it is low enough.
+ if (encoder_memusages[i] <= memory_limit)
+ break;
- // Otherwise 1 MiB down and try again. I hope this
- // isn't too slow method for cases where the original
- // dict_size is very big.
- opt->dict_size -= UINT32_C(1) << 20;
- }
+ // Otherwise adjust it 1 MiB down and try again.
+ opt->dict_size -= UINT32_C(1) << 20;
+ }
- // Tell the user that we decreased the dictionary size.
- message(V_WARNING, _("Adjusted LZMA%c dictionary size "
- "from %s MiB to %s MiB to not exceed "
- "the memory usage limit of %s MiB"),
- filters[i].id == LZMA_FILTER_LZMA2
- ? '2' : '1',
- uint64_to_str(orig_dict_size >> 20, 0),
- uint64_to_str(opt->dict_size >> 20, 1),
- uint64_to_str(round_up_to_mib(memory_limit), 2));
+ // Tell the user that we decreased the dictionary size.
+ // The message is slightly different between the default
+ // filter chain (0) or and chains from --filtersX.
+ const char lzma_num = chains[i][j].id == LZMA_FILTER_LZMA2
+ ? '2' : '1';
+ const char *from_size = uint64_to_str(orig_dict_size >> 20, 0);
+ const char *to_size = uint64_to_str(opt->dict_size >> 20, 1);
+ const char *limit_size = uint64_to_str(round_up_to_mib(
+ memory_limit), 2);
+ if (i == 0)
+ message(V_WARNING, _("Adjusted LZMA%c dictionary "
+ "size from %s MiB to %s MiB to not exceed the "
+ "memory usage limit of %s MiB"),
+ lzma_num, from_size, to_size, limit_size);
+ else
+ message(V_WARNING, _("Adjusted LZMA%c dictionary size "
+ "for --filters%u from %s MiB to %s MiB to not "
+ "exceed the memory usage limit of %s MiB"),
+ lzma_num, i, from_size, to_size, limit_size);
+ }
#endif
return;
@@ -512,6 +817,13 @@ coder_init(file_pair *pair)
// These will be handled later in this function.
allow_trailing_input = false;
+ // Set the first filter chain. If the --block-list option is not
+ // used then use the default filter chain (chains[0]).
+ // Otherwise, use first filter chain from the block list.
+ lzma_filter *active_filters = opt_block_list == NULL
+ ? chains[0]
+ : chains[opt_block_list[0].chain_num];
+
if (opt_mode == MODE_COMPRESS) {
#ifdef HAVE_ENCODERS
switch (opt_format) {
@@ -522,17 +834,19 @@ coder_init(file_pair *pair)
case FORMAT_XZ:
# ifdef MYTHREAD_ENABLED
+ mt_options.filters = active_filters;
if (hardware_threads_is_mt())
ret = lzma_stream_encoder_mt(
&strm, &mt_options);
else
# endif
ret = lzma_stream_encoder(
- &strm, filters, check);
+ &strm, active_filters, check);
break;
case FORMAT_LZMA:
- ret = lzma_alone_encoder(&strm, filters[0].options);
+ ret = lzma_alone_encoder(&strm,
+ active_filters[0].options);
break;
# ifdef HAVE_LZIP_DECODER
@@ -544,7 +858,7 @@ coder_init(file_pair *pair)
# endif
case FORMAT_RAW:
- ret = lzma_raw_encoder(&strm, filters);
+ ret = lzma_raw_encoder(&strm, active_filters);
break;
}
#endif
@@ -668,7 +982,7 @@ coder_init(file_pair *pair)
case FORMAT_RAW:
// Memory usage has already been checked in
// coder_set_compression_settings().
- ret = lzma_raw_decoder(&strm, filters);
+ ret = lzma_raw_decoder(&strm, active_filters);
break;
}
@@ -716,6 +1030,7 @@ coder_init(file_pair *pair)
}
+#ifdef HAVE_ENCODERS
/// Resolve conflicts between opt_block_size and opt_block_list in single
/// threaded mode. We want to default to opt_block_list, except when it is
/// larger than opt_block_size. If this is the case for the current Block
@@ -746,12 +1061,39 @@ split_block(uint64_t *block_remaining,
} else {
// The Block at *list_pos has been finished. Go to the next
- // entry in the list. If the end of the list has been reached,
- // reuse the size of the last Block.
- if (opt_block_list[*list_pos + 1] != 0)
+ // entry in the list. If the end of the list has been
+ // reached, reuse the size and filters of the last Block.
+ if (opt_block_list[*list_pos + 1].size != 0) {
++*list_pos;
- *block_remaining = opt_block_list[*list_pos];
+ // Update the filters if needed.
+ if (opt_block_list[*list_pos - 1].chain_num
+ != opt_block_list[*list_pos].chain_num) {
+ const unsigned chain_num
+ = opt_block_list[*list_pos].chain_num;
+ const lzma_filter *next = chains[chain_num];
+ const lzma_ret ret = lzma_filters_update(
+ &strm, next);
+
+ if (ret != LZMA_OK) {
+ // This message is only possible if
+ // the filter chain has unsupported
+ // options since the filter chain is
+ // validated using
+ // lzma_raw_encoder_memusage() or
+ // lzma_stream_encoder_mt_memusage().
+ // Some options are not validated until
+ // the encoders are initialized.
+ message_fatal(
+ _("Error changing to "
+ "filter chain %u: %s"),
+ chain_num,
+ message_strm(ret));
+ }
+ }
+ }
+
+ *block_remaining = opt_block_list[*list_pos].size;
// If in single-threaded mode, split up the Block if needed.
// This is not needed in multi-threaded mode because liblzma
@@ -764,6 +1106,7 @@ split_block(uint64_t *block_remaining,
}
}
}
+#endif
static bool
@@ -796,6 +1139,7 @@ coder_normal(file_pair *pair)
// Assume that something goes wrong.
bool success = false;
+#ifdef HAVE_ENCODERS
// block_remaining indicates how many input bytes to encode before
// finishing the current .xz Block. The Block size is set with
// --block-size=SIZE and --block-list. They have an effect only when
@@ -829,15 +1173,18 @@ coder_normal(file_pair *pair)
// output is still not identical because in single-threaded
// mode the size info isn't written into Block Headers.
if (opt_block_list != NULL) {
- if (block_remaining < opt_block_list[list_pos]) {
+ if (block_remaining < opt_block_list[list_pos].size) {
assert(!hardware_threads_is_mt());
- next_block_remaining = opt_block_list[list_pos]
+ next_block_remaining =
+ opt_block_list[list_pos].size
- block_remaining;
} else {
- block_remaining = opt_block_list[list_pos];
+ block_remaining =
+ opt_block_list[list_pos].size;
}
}
}
+#endif
strm.next_out = out_buf.u8;
strm.avail_out = IO_BUFFER_SIZE;
@@ -847,17 +1194,22 @@ coder_normal(file_pair *pair)
// flushing or finishing.
if (strm.avail_in == 0 && action == LZMA_RUN) {
strm.next_in = in_buf.u8;
- strm.avail_in = io_read(pair, &in_buf,
- my_min(block_remaining,
- IO_BUFFER_SIZE));
+#ifdef HAVE_ENCODERS
+ const size_t read_size = my_min(block_remaining,
+ IO_BUFFER_SIZE);
+#else
+ const size_t read_size = IO_BUFFER_SIZE;
+#endif
+ strm.avail_in = io_read(pair, &in_buf, read_size);
if (strm.avail_in == SIZE_MAX)
break;
if (pair->src_eof) {
action = LZMA_FINISH;
-
- } else if (block_remaining != UINT64_MAX) {
+ }
+#ifdef HAVE_ENCODERS
+ else if (block_remaining != UINT64_MAX) {
// Start a new Block after every
// opt_block_size bytes of input.
block_remaining -= strm.avail_in;
@@ -867,6 +1219,7 @@ coder_normal(file_pair *pair)
if (action == LZMA_RUN && pair->flush_needed)
action = LZMA_SYNC_FLUSH;
+#endif
}
// Let liblzma do the actual work.
@@ -878,6 +1231,7 @@ coder_normal(file_pair *pair)
break;
}
+#ifdef HAVE_ENCODERS
if (ret == LZMA_STREAM_END && (action == LZMA_SYNC_FLUSH
|| action == LZMA_FULL_BARRIER)) {
if (action == LZMA_SYNC_FLUSH) {
@@ -907,8 +1261,9 @@ coder_normal(file_pair *pair)
// Start a new Block after LZMA_FULL_FLUSH or continue
// the same block after LZMA_SYNC_FLUSH.
action = LZMA_RUN;
-
- } else if (ret != LZMA_OK) {
+ } else
+#endif
+ if (ret != LZMA_OK) {
// Determine if the return value indicates that we
// won't continue coding. LZMA_NO_CHECK would be
// here too if LZMA_TELL_ANY_CHECK was used.
@@ -1103,6 +1458,16 @@ coder_run(const char *filename)
extern void
coder_free(void)
{
+ // Free starting from the second filter chain since the default
+ // filter chain may have its options set from a static variable
+ // in coder_set_compression_settings(). Since this is only run in
+ // debug mode and will be freed when the process ends anyway, we
+ // don't worry about freeing it.
+ for (uint32_t i = 1; i < ARRAY_SIZE(chains); i++) {
+ if (chains_used_mask & (1U << i))
+ lzma_filters_free(chains[i], NULL);
+ }
+
lzma_end(&strm);
return;
}
diff --git a/src/xz/coder.h b/src/xz/coder.h
index b4f43a2..96755f3 100644
--- a/src/xz/coder.h
+++ b/src/xz/coder.h
@@ -1,12 +1,12 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file coder.h
/// \brief Compresses or uncompresses a file
//
-// Author: Lasse Collin
-//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
+// Authors: Lasse Collin
+// Jia Tan
//
///////////////////////////////////////////////////////////////////////////////
@@ -30,6 +30,16 @@ enum format_type {
};
+/// Array of these hold the entries specified with --block-list.
+typedef struct {
+ /// Uncompressed size of the Block
+ uint64_t size;
+
+ /// Filter chain to use for this Block (chains[chain_num])
+ unsigned chain_num;
+} block_list_entry;
+
+
/// Operation mode of the command line tool. This is set in args.c and read
/// in several files.
extern enum operation_mode opt_mode;
@@ -50,9 +60,21 @@ extern bool opt_single_stream;
/// of input. This has an effect only when compressing to the .xz format.
extern uint64_t opt_block_size;
-/// This is non-NULL if --block-list was used. This contains the Block sizes
-/// as an array that is terminated with 0.
-extern uint64_t *opt_block_list;
+/// List of block size and filter chain pointer pairs.
+extern block_list_entry *opt_block_list;
+
+/// Size of the largest Block that was specified in --block-list.
+/// This is used to limit the block_size option of multithreaded encoder.
+/// It's waste of memory to specify a too large block_size and reducing
+/// it might even allow using more threads in some cases.
+///
+/// NOTE: If the last entry in --block-list is the special value of 0
+/// (which gets converted to UINT64_MAX), it counts here as UINT64_MAX too.
+/// This way the multithreaded encoder's Block size won't be reduced.
+extern uint64_t block_list_largest;
+
+/// Bitmask indicating which filter chains we specified in --block-list.
+extern uint32_t block_list_chain_mask;
/// Set the integrity check type used when compressing
extern void coder_set_check(lzma_check check);
@@ -77,3 +99,9 @@ extern void coder_run(const char *filename);
/// Free the memory allocated for the coder and kill the worker threads.
extern void coder_free(void);
#endif
+
+/// Create filter chain from string
+extern void coder_add_filters_from_str(const char *filter_str);
+
+/// Add or overwrite a filter that can be used by the block-list.
+extern void coder_add_block_filters(const char *str, size_t slot);
diff --git a/src/xz/file_io.c b/src/xz/file_io.c
index 2828029..678a9a5 100644
--- a/src/xz/file_io.c
+++ b/src/xz/file_io.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file file_io.c
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
@@ -29,15 +28,33 @@ static bool warn_fchown;
# include <utime.h>
#endif
-#ifdef HAVE_CAPSICUM
-# ifdef HAVE_SYS_CAPSICUM_H
-# include <sys/capsicum.h>
+#include "tuklib_open_stdxxx.h"
+
+#ifdef _MSC_VER
+# ifdef _WIN64
+ typedef __int64 ssize_t;
# else
-# include <sys/capability.h>
+ typedef int ssize_t;
# endif
-#endif
-#include "tuklib_open_stdxxx.h"
+ typedef int mode_t;
+# define S_IRUSR _S_IREAD
+# define S_IWUSR _S_IWRITE
+
+# define setmode _setmode
+# define open _open
+# define close _close
+# define lseek _lseeki64
+# define unlink _unlink
+
+ // The casts are to silence warnings.
+ // The sizes are known to be small enough.
+# define read(fd, buf, size) _read(fd, buf, (unsigned int)(size))
+# define write(fd, buf, size) _write(fd, buf, (unsigned int)(size))
+
+# define S_ISDIR(m) (((m) & _S_IFMT) == _S_IFDIR)
+# define S_ISREG(m) (((m) & _S_IFMT) == _S_IFREG)
+#endif
#ifndef O_BINARY
# define O_BINARY 0
@@ -66,11 +83,6 @@ typedef enum {
/// If true, try to create sparse files when decompressing.
static bool try_sparse = true;
-#ifdef ENABLE_SANDBOX
-/// True if the conditions for sandboxing (described in main()) have been met.
-static bool sandbox_allowed = false;
-#endif
-
#ifndef TUKLIB_DOSLIKE
/// File status flags of standard input. This is used by io_open_src()
/// and io_close_src().
@@ -155,105 +167,6 @@ io_no_sparse(void)
}
-#ifdef ENABLE_SANDBOX
-extern void
-io_allow_sandbox(void)
-{
- sandbox_allowed = true;
- return;
-}
-
-
-/// Enables operating-system-specific sandbox if it is possible.
-/// src_fd is the file descriptor of the input file.
-static void
-io_sandbox_enter(int src_fd)
-{
- if (!sandbox_allowed) {
- // This message is more often annoying than useful so
- // it's commented out. It can be useful when developing
- // the sandboxing code.
- //message(V_DEBUG, _("Sandbox is disabled due "
- // "to incompatible command line arguments"));
- return;
- }
-
- const char dummy_str[] = "x";
-
- // Try to ensure that both libc and xz locale files have been
- // loaded when NLS is enabled.
- snprintf(NULL, 0, "%s%s", _(dummy_str), strerror(EINVAL));
-
- // Try to ensure that iconv data files needed for handling multibyte
- // characters have been loaded. This is needed at least with glibc.
- tuklib_mbstr_width(dummy_str, NULL);
-
-#ifdef HAVE_CAPSICUM
- // Capsicum needs FreeBSD 10.0 or later.
- cap_rights_t rights;
-
- if (cap_enter())
- goto error;
-
- if (cap_rights_limit(src_fd, cap_rights_init(&rights,
- CAP_EVENT, CAP_FCNTL, CAP_LOOKUP, CAP_READ, CAP_SEEK)))
- goto error;
-
- if (src_fd != STDIN_FILENO && cap_rights_limit(
- STDIN_FILENO, cap_rights_clear(&rights)))
- goto error;
-
- if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights,
- CAP_EVENT, CAP_FCNTL, CAP_FSTAT, CAP_LOOKUP,
- CAP_WRITE, CAP_SEEK)))
- goto error;
-
- if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights,
- CAP_WRITE)))
- goto error;
-
- if (cap_rights_limit(user_abort_pipe[0], cap_rights_init(&rights,
- CAP_EVENT)))
- goto error;
-
- if (cap_rights_limit(user_abort_pipe[1], cap_rights_init(&rights,
- CAP_WRITE)))
- goto error;
-
-#elif defined(HAVE_PLEDGE)
- // pledge() was introduced in OpenBSD 5.9.
- //
- // main() unconditionally calls pledge() with fairly relaxed
- // promises which work in all situations. Here we make the
- // sandbox more strict.
- if (pledge("stdio", ""))
- goto error;
-
- (void)src_fd;
-
-#else
-# error ENABLE_SANDBOX is defined but no sandboxing method was found.
-#endif
-
- // This message is annoying in xz -lvv.
- //message(V_DEBUG, _("Sandbox was successfully enabled"));
- return;
-
-error:
-#ifdef HAVE_CAPSICUM
- // If a kernel is configured without capability mode support or
- // used in an emulator that does not implement the capability
- // system calls, then the Capsicum system calls will fail and set
- // errno to ENOSYS. In that case xz will silently run without
- // the sandbox.
- if (errno == ENOSYS)
- return;
-#endif
- message_fatal(_("Failed to enable the sandbox"));
-}
-#endif // ENABLE_SANDBOX
-
-
#ifndef TUKLIB_DOSLIKE
/// \brief Waits for input or output to become available or for a signal
///
@@ -407,7 +320,7 @@ io_copy_attrs(const file_pair *pair)
message_warning(_("%s: Cannot set the file group: %s"),
pair->dest_name, strerror(errno));
// We can still safely copy some additional permissions:
- // `group' must be at least as strict as `other' and
+ // 'group' must be at least as strict as 'other' and
// also vice versa.
//
// NOTE: After this, the owner of the source file may
@@ -809,7 +722,8 @@ io_open_src(const char *src_name)
#ifdef ENABLE_SANDBOX
if (!error)
- io_sandbox_enter(pair.src_fd);
+ sandbox_enable_strict_if_allowed(pair.src_fd,
+ user_abort_pipe[0], user_abort_pipe[1]);
#endif
return error ? NULL : &pair;
diff --git a/src/xz/file_io.h b/src/xz/file_io.h
index 6992efa..ae7e2f3 100644
--- a/src/xz/file_io.h
+++ b/src/xz/file_io.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file file_io.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
// Some systems have suboptimal BUFSIZ. Use a bit bigger value on them.
@@ -18,6 +17,16 @@
# define IO_BUFFER_SIZE (BUFSIZ & ~7U)
#endif
+#ifdef _MSC_VER
+ // The first one renames both "struct stat" -> "struct _stat64"
+ // and stat() -> _stat64(). The documentation mentions only
+ // "struct __stat64", not "struct _stat64", but the latter
+ // works too.
+# define stat _stat64
+# define fstat _fstat64
+# define off_t __int64
+#endif
+
/// is_sparse() accesses the buffer as uint64_t for maximum speed.
/// The u32 and u64 members must only be access through this union
@@ -90,12 +99,6 @@ extern void io_write_to_user_abort_pipe(void);
extern void io_no_sparse(void);
-#ifdef ENABLE_SANDBOX
-/// \brief main() calls this if conditions for sandboxing have been met.
-extern void io_allow_sandbox(void);
-#endif
-
-
/// \brief Open the source file
extern file_pair *io_open_src(const char *src_name);
diff --git a/src/xz/hardware.c b/src/xz/hardware.c
index c694882..952652f 100644
--- a/src/xz/hardware.c
+++ b/src/xz/hardware.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file hardware.c
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
@@ -15,7 +14,7 @@
/// Maximum number of worker threads. This can be set with
/// the --threads=NUM command line option.
-static uint32_t threads_max = 1;
+static uint32_t threads_max;
/// True when the number of threads is automatically determined based
/// on the available hardware threads.
@@ -334,5 +333,9 @@ hardware_init(void)
memlimit_mt_default = mem_ceiling;
#endif
+ // Enable threaded mode by default. xz 5.4.x and older
+ // used single-threaded mode by default.
+ hardware_threads_set(0);
+
return;
}
diff --git a/src/xz/hardware.h b/src/xz/hardware.h
index a67b26e..25b351e 100644
--- a/src/xz/hardware.h
+++ b/src/xz/hardware.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file hardware.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
/// Initialize some hardware-specific variables, which are needed by other
diff --git a/src/xz/list.c b/src/xz/list.c
index 86c3a76..ca9cf03 100644
--- a/src/xz/list.c
+++ b/src/xz/list.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file list.c
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
@@ -544,11 +543,21 @@ parse_block_header(file_pair *pair, const lzma_index_iter *iter,
xfi->memusage_max = bhi->memusage;
// Determine the minimum XZ Utils version that supports this Block.
+ // - RISC-V filter needs 5.6.0.
//
// - ARM64 filter needs 5.4.0.
//
// - 5.0.0 doesn't support empty LZMA2 streams and thus empty
// Blocks that use LZMA2. This decoder bug was fixed in 5.0.2.
+ if (xfi->min_version < 50060002U) {
+ for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) {
+ if (filters[i].id == LZMA_FILTER_RISCV) {
+ xfi->min_version = 50060002U;
+ break;
+ }
+ }
+ }
+
if (xfi->min_version < 50040002U) {
for (size_t i = 0; filters[i].id != LZMA_VLI_UNKNOWN; ++i) {
if (filters[i].id == LZMA_FILTER_ARM64) {
@@ -1266,10 +1275,22 @@ list_totals(void)
extern void
list_file(const char *filename)
{
- if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO)
- message_fatal(_("--list works only on .xz files "
+ if (opt_format != FORMAT_XZ && opt_format != FORMAT_AUTO) {
+ // The 'lzmainfo' message is printed only when --format=lzma
+ // is used (it is implied if using "lzma" as the command
+ // name). Thus instead of using message_fatal(), print
+ // the messages separately and then call tuklib_exit()
+ // like message_fatal() does.
+ message(V_ERROR, _("--list works only on .xz files "
"(--format=xz or --format=auto)"));
+ if (opt_format == FORMAT_LZMA)
+ message(V_ERROR,
+ _("Try 'lzmainfo' with .lzma files."));
+
+ tuklib_exit(E_ERROR, E_ERROR, false);
+ }
+
message_filename(filename);
if (filename == stdin_filename) {
diff --git a/src/xz/list.h b/src/xz/list.h
index a4c6ec7..805880d 100644
--- a/src/xz/list.h
+++ b/src/xz/list.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file list.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
/// \brief List information about the given .xz file
diff --git a/src/xz/main.c b/src/xz/main.c
index c9c3dec..71b5ef7 100644
--- a/src/xz/main.c
+++ b/src/xz/main.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file main.c
@@ -5,14 +7,12 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
#include <ctype.h>
+
/// Exit status to use. This can be changed with set_exit_status().
static enum exit_status_type exit_status = E_SUCCESS;
@@ -119,8 +119,8 @@ read_name(const args_info *args)
// newlines.
message_error(_("%s: Null character found when "
"reading filenames; maybe you meant "
- "to use `--files0' instead "
- "of `--files'?"), args->files_name);
+ "to use '--files0' instead "
+ "of '--files'?"), args->files_name);
return NULL;
}
@@ -142,35 +142,38 @@ read_name(const args_info *args)
int
main(int argc, char **argv)
{
-#ifdef HAVE_PLEDGE
- // OpenBSD's pledge(2) sandbox
- //
- // Unconditionally enable sandboxing with fairly relaxed promises.
- // This is still way better than having no sandbox at all. :-)
- // More strict promises will be made later in file_io.c if possible.
- if (pledge("stdio rpath wpath cpath fattr", "")) {
- // Don't translate the string or use message_fatal() as
- // those haven't been initialized yet.
- fprintf(stderr, "%s: Failed to enable the sandbox\n", argv[0]);
- return E_ERROR;
- }
-#endif
-
#if defined(_WIN32) && !defined(__CYGWIN__)
InitializeCriticalSection(&exit_status_cs);
#endif
- // Set up the progname variable.
+ // Set up the progname variable needed for messages.
tuklib_progname_init(argv);
// Initialize the file I/O. This makes sure that
// stdin, stdout, and stderr are something valid.
+ // This must be done before we might open any files
+ // even indirectly like locale and gettext initializations.
io_init();
+#ifdef ENABLE_SANDBOX
+ // Enable such sandboxing that can always be enabled.
+ // This requires that progname has been set up.
+ // It's also good that io_init() has been called because it
+ // might need to do things that the initial sandbox won't allow.
+ // Otherwise this should be called as early as possible.
+ //
+ // NOTE: Calling this before tuklib_gettext_init() means that
+ // translated error message won't be available if sandbox
+ // initialization fails. However, sandbox_init() shouldn't
+ // fail and this order simply feels better.
+ sandbox_init();
+#endif
+
// Set up the locale and message translations.
tuklib_gettext_init(PACKAGE, LOCALEDIR);
- // Initialize handling of error/warning/other messages.
+ // Initialize progress message handling. It's not always needed
+ // but it's simpler to do this unconditionally.
message_init();
// Set hardware-dependent default values. These can be overridden
@@ -220,21 +223,41 @@ main(int argc, char **argv)
signals_init();
#ifdef ENABLE_SANDBOX
- // Set a flag that sandboxing is allowed if all these are true:
- // - --files or --files0 wasn't used.
- // - There is exactly one input file or we are reading from stdin.
- // - We won't create any files: output goes to stdout or --test
- // or --list was used. Note that --test implies opt_stdout = true
- // but --list doesn't.
+ // Read-only sandbox can be enabled if we won't create or delete
+ // any files:
//
- // This is obviously not ideal but it was easy to implement and
- // it covers the most common use cases.
+ // - --stdout, --test, or --list was used. Note that --test
+ // implies opt_stdout = true but --list doesn't.
//
- // TODO: Make sandboxing work for other situations too.
- if (args.files_name == NULL && args.arg_count == 1
- && (opt_stdout || strcmp("-", args.arg_names[0]) == 0
- || opt_mode == MODE_LIST))
- io_allow_sandbox();
+ // - Output goes to stdout because --files or --files0 wasn't used
+ // and no arguments were given on the command line or the
+ // arguments are all "-" (indicating standard input).
+ bool to_stdout_only = opt_stdout || opt_mode == MODE_LIST;
+ if (!to_stdout_only && args.files_name == NULL) {
+ // If all of the filenames provided are "-" (more than one
+ // "-" could be specified), then we are only going to be
+ // writing to standard output. Note that if no filename args
+ // were provided, args.c puts a single "-" in arg_names[0].
+ to_stdout_only = true;
+
+ for (unsigned i = 0; i < args.arg_count; ++i) {
+ if (strcmp("-", args.arg_names[i]) != 0) {
+ to_stdout_only = false;
+ break;
+ }
+ }
+ }
+
+ if (to_stdout_only) {
+ sandbox_enable_read_only();
+
+ // Allow strict sandboxing if we are processing exactly one
+ // file to standard output. This requires that --files or
+ // --files0 wasn't specified (an unknown number of filenames
+ // could be provided that way).
+ if (args.files_name == NULL && args.arg_count == 1)
+ sandbox_allow_strict();
+ }
#endif
// coder_run() handles compression, decompression, and testing.
diff --git a/src/xz/main.h b/src/xz/main.h
index 323f2f7..a8a1b45 100644
--- a/src/xz/main.h
+++ b/src/xz/main.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file main.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
/// Possible exit status values. These are the same as used by gzip and bzip2.
diff --git a/src/xz/message.c b/src/xz/message.c
index abf30ad..deafdb4 100644
--- a/src/xz/message.c
+++ b/src/xz/message.c
@@ -1,12 +1,12 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file message.c
/// \brief Printing messages
//
-// Author: Lasse Collin
-//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
+// Authors: Lasse Collin
+// Jia Tan
//
///////////////////////////////////////////////////////////////////////////////
@@ -42,7 +42,7 @@ static bool current_filename_printed = false;
/// True if we should print progress indicator and update it automatically
/// if also verbose >= V_VERBOSE.
-static bool progress_automatic;
+static bool progress_automatic = false;
/// True if message_progress_start() has been called but
/// message_progress_end() hasn't been called yet.
@@ -119,26 +119,7 @@ message_init(void)
// exception, even if --verbose was not used, user can send SIGALRM
// to make us print progress information once without automatic
// updating.
- progress_automatic = isatty(STDERR_FILENO);
-
- // Commented out because COLUMNS is rarely exported to environment.
- // Most users have at least 80 columns anyway, let's think something
- // fancy here if enough people complain.
-/*
- if (progress_automatic) {
- // stderr is a terminal. Check the COLUMNS environment
- // variable to see if the terminal is wide enough. If COLUMNS
- // doesn't exist or it has some unparsable value, we assume
- // that the terminal is wide enough.
- const char *columns_str = getenv("COLUMNS");
- if (columns_str != NULL) {
- char *endptr;
- const long columns = strtol(columns_str, &endptr, 10);
- if (*endptr != '\0' || columns < 80)
- progress_automatic = false;
- }
- }
-*/
+ progress_automatic = is_tty(STDERR_FILENO);
#ifdef SIGALRM
// Establish the signal handlers which set a flag to tell us that
@@ -932,7 +913,7 @@ message_try_help(void)
{
// Print this with V_WARNING instead of V_ERROR to prevent it from
// showing up when --quiet has been specified.
- message(V_WARNING, _("Try `%s --help' for more information."),
+ message(V_WARNING, _("Try '%s --help' for more information."),
progname);
return;
}
@@ -994,7 +975,7 @@ message_help(bool long_help)
" ignore possible remaining input data"));
puts(_(
" --no-sparse do not create sparse files when decompressing\n"
-" -S, --suffix=.SUF use the suffix `.SUF' on compressed files\n"
+" -S, --suffix=.SUF use the suffix '.SUF' on compressed files\n"
" --files[=FILE] read filenames to process from FILE; if FILE is\n"
" omitted, filenames are read from the standard input;\n"
" filenames must be terminated with the newline character\n"
@@ -1005,9 +986,9 @@ message_help(bool long_help)
puts(_("\n Basic file format and compression options:\n"));
puts(_(
" -F, --format=FMT file format to encode or decode; possible values are\n"
-" `auto' (default), `xz', `lzma', `lzip', and `raw'\n"
-" -C, --check=CHECK integrity check type: `none' (use with caution),\n"
-" `crc32', `crc64' (default), or `sha256'"));
+" 'auto' (default), 'xz', 'lzma', 'lzip', and 'raw'\n"
+" -C, --check=CHECK integrity check type: 'none' (use with caution),\n"
+" 'crc32', 'crc64' (default), or 'sha256'"));
puts(_(
" --ignore-check don't verify the integrity check when decompressing"));
}
@@ -1021,8 +1002,8 @@ message_help(bool long_help)
" does not affect decompressor memory requirements"));
puts(_(
-" -T, --threads=NUM use at most NUM threads; the default is 1; set to 0\n"
-" to use as many threads as there are processor cores"));
+" -T, --threads=NUM use at most NUM threads; the default is 0 which uses\n"
+" as many threads as there are processor cores"));
if (long_help) {
puts(_(
@@ -1030,9 +1011,11 @@ message_help(bool long_help)
" start a new .xz block after every SIZE bytes of input;\n"
" use this to set the block size for threaded compression"));
puts(_(
-" --block-list=SIZES\n"
+" --block-list=BLOCKS\n"
" start a new .xz block after the given comma-separated\n"
-" intervals of uncompressed data"));
+" intervals of uncompressed data; optionally, specify a\n"
+" filter chain number (0-9) followed by a ':' before the\n"
+" uncompressed data size"));
puts(_(
" --flush-timeout=TIMEOUT\n"
" when compressing, if more than TIMEOUT milliseconds has\n"
@@ -1057,6 +1040,23 @@ message_help(bool long_help)
puts(_(
"\n Custom filter chain for compression (alternative for using presets):"));
+ puts(_(
+"\n"
+" --filters=FILTERS set the filter chain using the liblzma filter string\n"
+" syntax; use --filters-help for more information"
+ ));
+
+ puts(_(
+" --filters1=FILTERS ... --filters9=FILTERS\n"
+" set additional filter chains using the liblzma filter\n"
+" string syntax to use with --block-list"
+ ));
+
+ puts(_(
+" --filters-help display more information about the liblzma filter string\n"
+" syntax and exit."
+ ));
+
#if defined(HAVE_ENCODER_LZMA1) || defined(HAVE_DECODER_LZMA1) \
|| defined(HAVE_ENCODER_LZMA2) || defined(HAVE_DECODER_LZMA2)
// TRANSLATORS: The word "literal" in "literal context bits"
@@ -1087,6 +1087,7 @@ message_help(bool long_help)
" --powerpc[=OPTS] PowerPC BCJ filter (big endian only)\n"
" --ia64[=OPTS] IA-64 (Itanium) BCJ filter\n"
" --sparc[=OPTS] SPARC BCJ filter\n"
+" --riscv[=OPTS] RISC-V BCJ filter\n"
" Valid OPTS for all BCJ filters:\n"
" start=NUM start offset for conversions (default=0)"));
@@ -1144,3 +1145,28 @@ message_help(bool long_help)
tuklib_exit(E_SUCCESS, E_ERROR, verbosity != V_SILENT);
}
+
+
+extern void
+message_filters_help(void)
+{
+ char *encoder_options;
+ if (lzma_str_list_filters(&encoder_options, LZMA_VLI_UNKNOWN,
+ LZMA_STR_ENCODER, NULL) != LZMA_OK)
+ message_bug();
+
+ if (!opt_robot) {
+ puts(_(
+"Filter chains are set using the --filters=FILTERS or\n"
+"--filters1=FILTERS ... --filters9=FILTERS options. Each filter in the chain\n"
+"can be separated by spaces or '--'. Alternatively a preset <0-9>[e] can be\n"
+"specified instead of a filter chain.\n"
+ ));
+
+ puts(_("The supported filters and their options are:"));
+ }
+
+ puts(encoder_options);
+
+ tuklib_exit(E_SUCCESS, E_ERROR, verbosity != V_SILENT);
+}
diff --git a/src/xz/message.h b/src/xz/message.h
index f608ec7..3f6e183 100644
--- a/src/xz/message.h
+++ b/src/xz/message.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file message.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
/// Verbosity levels
@@ -24,7 +23,10 @@ enum message_verbosity {
extern const int message_progress_sigs[];
-/// \brief Initializes the message functions
+/// \brief Initializes the progress message functions
+///
+/// message_fatal() and such can be called even before message_init()
+/// has been called.
///
/// If an error occurs, this function doesn't return.
///
@@ -111,6 +113,12 @@ tuklib_attr_noreturn
extern void message_help(bool long_help);
+/// Prints a help message specifically for using the --filters and
+/// --filtersX command line options.
+tuklib_attr_noreturn
+extern void message_filters_help(void);
+
+
/// \brief Set the total number of files to be processed
///
/// Standard input is counted as a file here. This is used when printing
diff --git a/src/xz/mytime.c b/src/xz/mytime.c
index 7e8a074..7d9a27d 100644
--- a/src/xz/mytime.c
+++ b/src/xz/mytime.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file mytime.c
@@ -5,14 +7,14 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
-#if defined(HAVE_CLOCK_GETTIME) && defined(HAVE_CLOCK_MONOTONIC)
+#if defined(MYTHREAD_VISTA) || defined(_MSC_VER)
+ // Nothing
+#elif defined(HAVE_CLOCK_GETTIME) \
+ && (!defined(__MINGW32__) || defined(MYTHREAD_POSIX))
# include <time.h>
#else
# include <sys/time.h>
@@ -20,7 +22,22 @@
uint64_t opt_flush_timeout = 0;
+// start_time holds the time when the (de)compression was started.
+// It's from mytime_now() and thus only useful for calculating relative
+// time differences (elapsed time). start_time is initialized by calling
+// mytime_set_start_time() and modified by mytime_sigtstp_handler().
+//
+// When mytime_sigtstp_handler() is used, start_time is made volatile.
+// I'm not sure if that is really required since access to it is guarded
+// by signals_block()/signals_unblock() since accessing an uint64_t isn't
+// atomic on all systems. But since the variable isn't accessed very
+// frequently making it volatile doesn't hurt.
+#ifdef USE_SIGTSTP_HANDLER
+static volatile uint64_t start_time;
+#else
static uint64_t start_time;
+#endif
+
static uint64_t next_flush;
@@ -30,16 +47,41 @@ static uint64_t next_flush;
static uint64_t
mytime_now(void)
{
-#if defined(HAVE_CLOCK_GETTIME) && defined(HAVE_CLOCK_MONOTONIC)
+#if defined(MYTHREAD_VISTA) || defined(_MSC_VER)
+ // Since there is no SIGALRM on Windows, this function gets
+ // called frequently when the progress indicator is in use.
+ // Progress indicator doesn't need high-resolution time.
+ // GetTickCount64() has very low overhead but needs at least WinVista.
+ //
+ // MinGW-w64 provides the POSIX functions clock_gettime() and
+ // gettimeofday() in a manner that allow xz to run on older
+ // than WinVista. If the threading method needs WinVista anyway,
+ // there's no reason to avoid a WinVista API here either.
+ return GetTickCount64();
+
+#elif defined(HAVE_CLOCK_GETTIME) \
+ && (!defined(__MINGW32__) || defined(MYTHREAD_POSIX))
+ // MinGW-w64: clock_gettime() is defined in winpthreads but we need
+ // nothing else from winpthreads (unless, for some odd reason, POSIX
+ // threading has been selected). By avoiding clock_gettime(), we
+ // avoid the dependency on libwinpthread-1.dll or the need to link
+ // against the static version. The downside is that the fallback
+ // method, gettimeofday(), doesn't provide monotonic time.
+ struct timespec tv;
+
+# ifdef HAVE_CLOCK_MONOTONIC
// If CLOCK_MONOTONIC was available at compile time but for some
// reason isn't at runtime, fallback to CLOCK_REALTIME which
// according to POSIX is mandatory for all implementations.
static clockid_t clk_id = CLOCK_MONOTONIC;
- struct timespec tv;
while (clock_gettime(clk_id, &tv))
clk_id = CLOCK_REALTIME;
+# else
+ clock_gettime(CLOCK_REALTIME, &tv);
+# endif
return (uint64_t)tv.tv_sec * 1000 + (uint64_t)(tv.tv_nsec / 1000000);
+
#else
struct timeval tv;
gettimeofday(&tv, NULL);
@@ -48,10 +90,49 @@ mytime_now(void)
}
+#ifdef USE_SIGTSTP_HANDLER
+extern void
+mytime_sigtstp_handler(int sig lzma_attribute((__unused__)))
+{
+ // Measure how long the process stays in the stopped state and add
+ // that amount to start_time. This way the the progress indicator
+ // won't count the stopped time as elapsed time and the estimated
+ // remaining time won't be confused by the time spent in the
+ // stopped state.
+ //
+ // FIXME? Is raising SIGSTOP the correct thing to do? POSIX.1-2017
+ // says that orphan processes shouldn't stop on SIGTSTP. So perhaps
+ // the most correct thing to do could be to revert to the default
+ // handler for SIGTSTP, unblock SIGTSTP, and then raise(SIGTSTP).
+ // It's quite a bit more complicated than just raising SIGSTOP though.
+ //
+ // The difference between raising SIGTSTP vs. SIGSTOP can be seen on
+ // the shell command line too by running "echo $?" after stopping
+ // a process but perhaps that doesn't matter.
+ const uint64_t t = mytime_now();
+ raise(SIGSTOP);
+ start_time += mytime_now() - t;
+ return;
+}
+#endif
+
+
extern void
mytime_set_start_time(void)
{
+#ifdef USE_SIGTSTP_HANDLER
+ // Block the signals when accessing start_time so that we cannot
+ // end up with a garbage value. start_time is volatile but access
+ // to it isn't atomic at least on 32-bit systems.
+ signals_block();
+#endif
+
start_time = mytime_now();
+
+#ifdef USE_SIGTSTP_HANDLER
+ signals_unblock();
+#endif
+
return;
}
@@ -59,7 +140,17 @@ mytime_set_start_time(void)
extern uint64_t
mytime_get_elapsed(void)
{
- return mytime_now() - start_time;
+#ifdef USE_SIGTSTP_HANDLER
+ signals_block();
+#endif
+
+ const uint64_t t = mytime_now() - start_time;
+
+#ifdef USE_SIGTSTP_HANDLER
+ signals_unblock();
+#endif
+
+ return t;
}
diff --git a/src/xz/mytime.h b/src/xz/mytime.h
index a7be2aa..6dfaeae 100644
--- a/src/xz/mytime.h
+++ b/src/xz/mytime.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file mytime.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
@@ -21,6 +20,12 @@
extern uint64_t opt_flush_timeout;
+#ifdef USE_SIGTSTP_HANDLER
+/// \brief Signal handler for SIGTSTP
+extern void mytime_sigtstp_handler(int sig);
+#endif
+
+
/// \brief Store the time when (de)compression was started
///
/// The start time is also stored as the time of the first flush.
diff --git a/src/xz/options.c b/src/xz/options.c
index 4d5e899..bc8bc1a 100644
--- a/src/xz/options.c
+++ b/src/xz/options.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file options.c
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
@@ -83,7 +82,7 @@ parse_options(const char *str, const option_map *opts,
*value++ = '\0';
if (value == NULL || value[0] == '\0')
- message_fatal(_("%s: Options must be `name=value' "
+ message_fatal(_("%s: Options must be 'name=value' "
"pairs separated with commas"), str);
// Look for the option name from the option map.
diff --git a/src/xz/options.h b/src/xz/options.h
index 61ec8d5..4a1314d 100644
--- a/src/xz/options.h
+++ b/src/xz/options.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file options.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
/// \brief Parser for Delta options
diff --git a/src/xz/private.h b/src/xz/private.h
index 6414bdb..b370472 100644
--- a/src/xz/private.h
+++ b/src/xz/private.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file private.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "sysdefs.h"
@@ -21,7 +20,10 @@
#include <signal.h>
#include <locale.h>
#include <stdio.h>
-#include <unistd.h>
+
+#ifndef _MSC_VER
+# include <unistd.h>
+#endif
#include "tuklib_gettext.h"
#include "tuklib_progname.h"
@@ -33,6 +35,10 @@
# include <windows.h>
#endif
+#ifdef _MSC_VER
+# define fileno _fileno
+#endif
+
#ifndef STDIN_FILENO
# define STDIN_FILENO (fileno(stdin))
#endif
@@ -45,8 +51,15 @@
# define STDERR_FILENO (fileno(stderr))
#endif
-#if defined(HAVE_CAPSICUM) || defined(HAVE_PLEDGE)
-# define ENABLE_SANDBOX 1
+// Handling SIGTSTP keeps time-keeping for progress indicator correct
+// if xz is stopped. It requires use of clock_gettime() as that is
+// async-signal safe in POSIX. Require also SIGALRM support since
+// on systems where SIGALRM isn't available, progress indicator code
+// polls the time and the SIGTSTP handling adds slight overhead to
+// that code. Most (all?) systems that have SIGTSTP also have SIGALRM
+// so this requirement won't exclude many systems.
+#if defined(HAVE_CLOCK_GETTIME) && defined(SIGTSTP) && defined(SIGALRM)
+# define USE_SIGTSTP_HANDLER 1
#endif
#include "main.h"
@@ -57,6 +70,7 @@
#include "hardware.h"
#include "file_io.h"
#include "options.h"
+#include "sandbox.h"
#include "signals.h"
#include "suffix.h"
#include "util.h"
diff --git a/src/xz/sandbox.c b/src/xz/sandbox.c
new file mode 100644
index 0000000..5bd2273
--- /dev/null
+++ b/src/xz/sandbox.c
@@ -0,0 +1,355 @@
+// SPDX-License-Identifier: 0BSD
+
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file sandbox.c
+/// \brief Sandbox support
+//
+// Author: Lasse Collin
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#include "private.h"
+
+
+#ifndef ENABLE_SANDBOX
+
+// Prevent an empty translation unit when no sandboxing is supported.
+typedef int dummy;
+
+#else
+
+/// If the conditions for strict sandboxing (described in main())
+/// have been met, sandbox_allow_strict() can be called to set this
+/// variable to true.
+static bool strict_sandbox_allowed = false;
+
+
+extern void
+sandbox_allow_strict(void)
+{
+ strict_sandbox_allowed = true;
+ return;
+}
+
+
+// Strict sandboxing prevents opening any files. This *tries* to ensure
+// that any auxiliary files that might be required are already open.
+//
+// Returns true if strict sandboxing is allowed, false otherwise.
+static bool
+prepare_for_strict_sandbox(void)
+{
+ if (!strict_sandbox_allowed)
+ return false;
+
+ const char dummy_str[] = "x";
+
+ // Try to ensure that both libc and xz locale files have been
+ // loaded when NLS is enabled.
+ snprintf(NULL, 0, "%s%s", _(dummy_str), strerror(EINVAL));
+
+ // Try to ensure that iconv data files needed for handling multibyte
+ // characters have been loaded. This is needed at least with glibc.
+ tuklib_mbstr_width(dummy_str, NULL);
+
+ return true;
+}
+
+#endif
+
+
+#if defined(HAVE_PLEDGE)
+
+///////////////
+// pledge(2) //
+///////////////
+
+#include <unistd.h>
+
+
+extern void
+sandbox_init(void)
+{
+ if (pledge("stdio rpath wpath cpath fattr", "")) {
+ // gettext hasn't been initialized yet so
+ // there's no point to call it here.
+ message_fatal("Failed to enable the sandbox");
+ }
+
+ return;
+}
+
+
+extern void
+sandbox_enable_read_only(void)
+{
+ // We will be opening files for reading but
+ // won't create or remove any files.
+ if (pledge("stdio rpath", ""))
+ message_fatal(_("Failed to enable the sandbox"));
+
+ return;
+}
+
+
+extern void
+sandbox_enable_strict_if_allowed(int src_fd lzma_attribute((__unused__)),
+ int pipe_event_fd lzma_attribute((__unused__)),
+ int pipe_write_fd lzma_attribute((__unused__)))
+{
+ if (!prepare_for_strict_sandbox())
+ return;
+
+ // All files that need to be opened have already been opened.
+ if (pledge("stdio", ""))
+ message_fatal(_("Failed to enable the sandbox"));
+
+ return;
+}
+
+
+#elif defined(HAVE_LINUX_LANDLOCK)
+
+//////////////
+// Landlock //
+//////////////
+
+#include <linux/landlock.h>
+#include <sys/syscall.h>
+#include <sys/prctl.h>
+
+
+// Highest Landlock ABI version supported by this file:
+// - For ABI versions 1-3 we don't need anything from <linux/landlock.h>
+// that isn't part of version 1.
+// - For ABI version 4 we need the larger struct landlock_ruleset_attr
+// with the handled_access_net member. That is bundled with the macros
+// LANDLOCK_ACCESS_NET_BIND_TCP and LANDLOCK_ACCESS_NET_CONNECT_TCP.
+#ifdef LANDLOCK_ACCESS_NET_BIND_TCP
+# define LANDLOCK_ABI_MAX 4
+#else
+# define LANDLOCK_ABI_MAX 3
+#endif
+
+
+/// Landlock ABI version supported by the kernel
+static int landlock_abi;
+
+
+// The required_rights should have those bits set that must not be restricted.
+// This function will then bitwise-and ~required_rights with a mask matching
+// the Landlock ABI version, leaving only those bits set that are supported
+// by the ABI and allowed to be restricted by the function argument.
+static void
+enable_landlock(uint64_t required_rights)
+{
+ assert(landlock_abi <= LANDLOCK_ABI_MAX);
+
+ if (landlock_abi <= 0)
+ return;
+
+ // We want to set all supported flags in handled_access_fs.
+ // This way the ruleset will initially forbid access to all
+ // actions that the available Landlock ABI version supports.
+ // Exceptions can be added using landlock_add_rule(2) to
+ // allow certain actions on certain files or directories.
+ //
+ // The same flag values are used on all archs. ABI v2 and v3
+ // both add one new flag.
+ //
+ // First in ABI v1: LANDLOCK_ACCESS_FS_EXECUTE = 1ULL << 0
+ // Last in ABI v1: LANDLOCK_ACCESS_FS_MAKE_SYM = 1ULL << 12
+ // Last in ABI v2: LANDLOCK_ACCESS_FS_REFER = 1ULL << 13
+ // Last in ABI v3: LANDLOCK_ACCESS_FS_TRUNCATE = 1ULL << 14
+ //
+ // This makes it simple to set the mask based on the ABI
+ // version and we don't need to care which flags are #defined
+ // in the installed <linux/landlock.h> for ABI versions 1-3.
+ const struct landlock_ruleset_attr attr = {
+ .handled_access_fs = ~required_rights
+ & ((1ULL << (12 + my_min(3, landlock_abi))) - 1),
+#if LANDLOCK_ABI_MAX >= 4
+ .handled_access_net = landlock_abi < 4 ? 0 :
+ (LANDLOCK_ACCESS_NET_BIND_TCP
+ | LANDLOCK_ACCESS_NET_CONNECT_TCP),
+#endif
+ };
+
+ const int ruleset_fd = syscall(SYS_landlock_create_ruleset,
+ &attr, sizeof(attr), 0U);
+ if (ruleset_fd < 0)
+ message_fatal(_("Failed to enable the sandbox"));
+
+ // All files we need should have already been opened. Thus,
+ // we don't need to add any rules using landlock_add_rule(2)
+ // before activating the sandbox.
+ //
+ // NOTE: It's possible that the hack prepare_for_strict_sandbox()
+ // isn't be good enough. It tries to get translations and
+ // libc-specific files loaded but if it's not good enough
+ // then perhaps a Landlock rule to allow reading from /usr
+ // and/or the xz installation prefix would be needed.
+ //
+ // prctl(PR_SET_NO_NEW_PRIVS, ...) was already called in
+ // sandbox_init() so we don't do it here again.
+ if (syscall(SYS_landlock_restrict_self, ruleset_fd, 0U) != 0)
+ message_fatal(_("Failed to enable the sandbox"));
+
+ return;
+}
+
+
+extern void
+sandbox_init(void)
+{
+ // Prevent the process from gaining new privileges. This must be done
+ // before landlock_restrict_self(2) but since we will never need new
+ // privileges, this call can be done here already.
+ //
+ // This is supported since Linux 3.5. Ignore the return value to
+ // keep compatibility with old kernels. landlock_restrict_self(2)
+ // will fail if the no_new_privs attribute isn't set, thus if prctl()
+ // fails here the error will still be detected when it matters.
+ (void)prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0);
+
+ // Get the highest Landlock ABI version supported by the kernel.
+ landlock_abi = syscall(SYS_landlock_create_ruleset,
+ (void *)NULL, 0, LANDLOCK_CREATE_RULESET_VERSION);
+
+ // The kernel might support a newer ABI than this file.
+ if (landlock_abi > LANDLOCK_ABI_MAX)
+ landlock_abi = LANDLOCK_ABI_MAX;
+
+ // These are all in ABI version 1 already. We don't need truncate
+ // rights because files are created with open() using O_EXCL and
+ // without O_TRUNC.
+ //
+ // LANDLOCK_ACCESS_FS_READ_DIR is included here to get a clear error
+ // message if xz is given a directory name. Without this permission
+ // the message would be "Permission denied" but with this permission
+ // it's "Is a directory, skipping". It could be worked around with
+ // stat()/lstat() but just giving this permission is simpler and
+ // shouldn't make the sandbox much weaker in practice.
+ const uint64_t required_rights
+ = LANDLOCK_ACCESS_FS_WRITE_FILE
+ | LANDLOCK_ACCESS_FS_READ_FILE
+ | LANDLOCK_ACCESS_FS_READ_DIR
+ | LANDLOCK_ACCESS_FS_REMOVE_FILE
+ | LANDLOCK_ACCESS_FS_MAKE_REG;
+
+ enable_landlock(required_rights);
+ return;
+}
+
+
+extern void
+sandbox_enable_read_only(void)
+{
+ // We will be opening files for reading but
+ // won't create or remove any files.
+ const uint64_t required_rights
+ = LANDLOCK_ACCESS_FS_READ_FILE
+ | LANDLOCK_ACCESS_FS_READ_DIR;
+ enable_landlock(required_rights);
+ return;
+}
+
+
+extern void
+sandbox_enable_strict_if_allowed(int src_fd lzma_attribute((__unused__)),
+ int pipe_event_fd lzma_attribute((__unused__)),
+ int pipe_write_fd lzma_attribute((__unused__)))
+{
+ if (!prepare_for_strict_sandbox())
+ return;
+
+ // Allow all restrictions that the kernel supports with the
+ // highest Landlock ABI version that the kernel or xz supports.
+ //
+ // NOTE: LANDLOCK_ACCESS_FS_READ_DIR isn't needed here because
+ // the only input file has already been opened.
+ enable_landlock(0);
+ return;
+}
+
+
+#elif defined(HAVE_CAP_RIGHTS_LIMIT)
+
+//////////////
+// Capsicum //
+//////////////
+
+#include <sys/capsicum.h>
+
+
+extern void
+sandbox_init(void)
+{
+ // Nothing to do.
+ return;
+}
+
+
+extern void
+sandbox_enable_read_only(void)
+{
+ // Nothing to do.
+ return;
+}
+
+
+extern void
+sandbox_enable_strict_if_allowed(
+ int src_fd, int pipe_event_fd, int pipe_write_fd)
+{
+ if (!prepare_for_strict_sandbox())
+ return;
+
+ // Capsicum needs FreeBSD 10.2 or later.
+ cap_rights_t rights;
+
+ if (cap_enter())
+ goto error;
+
+ if (cap_rights_limit(src_fd, cap_rights_init(&rights,
+ CAP_EVENT, CAP_FCNTL, CAP_LOOKUP, CAP_READ, CAP_SEEK)))
+ goto error;
+
+ // If not reading from stdin, remove all capabilities from it.
+ if (src_fd != STDIN_FILENO && cap_rights_limit(
+ STDIN_FILENO, cap_rights_clear(&rights)))
+ goto error;
+
+ if (cap_rights_limit(STDOUT_FILENO, cap_rights_init(&rights,
+ CAP_EVENT, CAP_FCNTL, CAP_FSTAT, CAP_LOOKUP,
+ CAP_WRITE, CAP_SEEK)))
+ goto error;
+
+ if (cap_rights_limit(STDERR_FILENO, cap_rights_init(&rights,
+ CAP_WRITE)))
+ goto error;
+
+ if (cap_rights_limit(pipe_event_fd, cap_rights_init(&rights,
+ CAP_EVENT)))
+ goto error;
+
+ if (cap_rights_limit(pipe_write_fd, cap_rights_init(&rights,
+ CAP_WRITE)))
+ goto error;
+
+ return;
+
+error:
+ // If a kernel is configured without capability mode support or
+ // used in an emulator that does not implement the capability
+ // system calls, then the Capsicum system calls will fail and set
+ // errno to ENOSYS. In that case xz will silently run without
+ // the sandbox.
+ if (errno == ENOSYS)
+ return;
+
+ message_fatal(_("Failed to enable the sandbox"));
+}
+
+#endif
diff --git a/src/xz/sandbox.h b/src/xz/sandbox.h
new file mode 100644
index 0000000..98b9862
--- /dev/null
+++ b/src/xz/sandbox.h
@@ -0,0 +1,43 @@
+// SPDX-License-Identifier: 0BSD
+
+///////////////////////////////////////////////////////////////////////////////
+//
+/// \file sandbox.h
+/// \brief Sandbox support
+//
+// Author: Lasse Collin
+//
+///////////////////////////////////////////////////////////////////////////////
+
+#if defined(HAVE_PLEDGE) || defined(HAVE_LINUX_LANDLOCK) \
+ || defined(HAVE_CAP_RIGHTS_LIMIT)
+# define ENABLE_SANDBOX 1
+#endif
+
+
+/// \brief Enables early sandboxing that can always be enabled
+///
+/// This requires that tuklib_progname() and io_init() have been called.
+extern void sandbox_init(void);
+
+
+/// \brief Enable sandboxing that only allows opening files for reading
+extern void sandbox_enable_read_only(void);
+
+
+/// \brief Tell sandboxing code that strict sandboxing can be used
+///
+/// This function only sets a flag which will be read by
+/// sandbox_enable_strict_if_allowed().
+extern void sandbox_allow_strict(void);
+
+
+/// \brief Enable sandboxing that allows reading from one file
+///
+/// This does nothing if sandbox_allow_strict() hasn't been called.
+///
+/// \param src_fd File descriptor open for reading
+/// \param pipe_event_fd user_abort_pipe[0] from file_io.c
+/// \param pipe_write_fd user_abort_pipe[1] from file_io.c
+extern void sandbox_enable_strict_if_allowed(
+ int src_fd, int pipe_event_fd, int pipe_write_fd);
diff --git a/src/xz/signals.c b/src/xz/signals.c
index 7aef463..20f03be 100644
--- a/src/xz/signals.c
+++ b/src/xz/signals.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file signals.c
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
@@ -50,6 +49,10 @@ signal_handler(int sig)
}
+#ifdef __APPLE__
+# pragma GCC diagnostic push
+# pragma GCC diagnostic ignored "-Wsign-conversion"
+#endif
extern void
signals_init(void)
{
@@ -82,6 +85,11 @@ signals_init(void)
sigaddset(&hooked_signals, message_progress_sigs[i]);
#endif
+#ifdef USE_SIGTSTP_HANDLER
+ // Add the SIGTSTP handler from mytime.c to hooked_signals.
+ sigaddset(&hooked_signals, SIGTSTP);
+#endif
+
// Using "my_sa" because "sa" may conflict with a sockaddr variable
// from system headers on Solaris.
struct sigaction my_sa;
@@ -96,10 +104,11 @@ signals_init(void)
my_sa.sa_flags = 0;
my_sa.sa_handler = &signal_handler;
+ struct sigaction old;
+
for (size_t i = 0; i < ARRAY_SIZE(sigs); ++i) {
// If the parent process has left some signals ignored,
// we don't unignore them.
- struct sigaction old;
if (sigaction(sigs[i], NULL, &old) == 0
&& old.sa_handler == SIG_IGN)
continue;
@@ -109,10 +118,22 @@ signals_init(void)
message_signal_handler();
}
+#ifdef USE_SIGTSTP_HANDLER
+ if (!(sigaction(SIGTSTP, NULL, &old) == 0
+ && old.sa_handler == SIG_IGN)) {
+ my_sa.sa_handler = &mytime_sigtstp_handler;
+ if (sigaction(SIGTSTP, &my_sa, NULL))
+ message_signal_handler();
+ }
+#endif
+
signals_are_initialized = true;
return;
}
+#ifdef __APPLE__
+# pragma GCC diagnostic pop
+#endif
#ifndef __VMS
diff --git a/src/xz/signals.h b/src/xz/signals.h
index 5b125e0..629335d 100644
--- a/src/xz/signals.h
+++ b/src/xz/signals.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file signals.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
/// If this is true, we will clean up the possibly incomplete output file,
diff --git a/src/xz/suffix.c b/src/xz/suffix.c
index 6ce9787..1d548e4 100644
--- a/src/xz/suffix.c
+++ b/src/xz/suffix.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file suffix.c
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
@@ -21,7 +20,13 @@
# ifdef HAVE_STRINGS_H
# include <strings.h>
# endif
-# define strcmp strcasecmp
+# ifdef _MSC_VER
+# define suffix_strcmp _stricmp
+# else
+# define suffix_strcmp strcasecmp
+# endif
+#else
+# define suffix_strcmp strcmp
#endif
@@ -98,7 +103,7 @@ test_suffix(const char *suffix, const char *src_name, size_t src_len)
|| is_dir_sep(src_name[src_len - suffix_len - 1]))
return 0;
- if (strcmp(suffix, src_name + src_len - suffix_len) == 0)
+ if (suffix_strcmp(suffix, src_name + src_len - suffix_len) == 0)
return src_len - suffix_len;
return 0;
@@ -178,7 +183,7 @@ uncompressed_name(const char *src_name, const size_t src_len)
static void
msg_suffix(const char *src_name, const char *suffix)
{
- message_warning(_("%s: File already has `%s' suffix, skipping"),
+ message_warning(_("%s: File already has '%s' suffix, skipping"),
src_name, suffix);
return;
}
diff --git a/src/xz/suffix.h b/src/xz/suffix.h
index 135e905..f59e312 100644
--- a/src/xz/suffix.h
+++ b/src/xz/suffix.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file suffix.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
/// \brief Get the name of the destination file
diff --git a/src/xz/util.c b/src/xz/util.c
index 6ab4c2d..0d339ae 100644
--- a/src/xz/util.c
+++ b/src/xz/util.c
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file util.c
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
#include "private.h"
@@ -148,8 +147,8 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max)
if (multiplier == 0) {
message(V_ERROR, _("%s: Invalid multiplier suffix"),
value - 1);
- message_fatal(_("Valid suffixes are `KiB' (2^10), "
- "`MiB' (2^20), and `GiB' (2^30)."));
+ message_fatal(_("Valid suffixes are 'KiB' (2^10), "
+ "'MiB' (2^20), and 'GiB' (2^30)."));
}
// Don't overflow here either.
@@ -165,7 +164,7 @@ str_to_uint64(const char *name, const char *value, uint64_t min, uint64_t max)
return result;
error:
- message_fatal(_("Value of the option `%s' must be in the range "
+ message_fatal(_("Value of the option '%s' must be in the range "
"[%" PRIu64 ", %" PRIu64 "]"),
name, min, max);
}
@@ -262,9 +261,30 @@ my_snprintf(char **pos, size_t *left, const char *fmt, ...)
extern bool
+is_tty(int fd)
+{
+#if defined(_WIN32) && !defined(__CYGWIN__)
+ // There is no need to check if handle == INVALID_HANDLE_VALUE
+ // because it will return false anyway when used in GetConsoleMode().
+ // The resulting HANDLE is owned by the file descriptor.
+ // The HANDLE must not be closed here.
+ intptr_t handle = _get_osfhandle(fd);
+ DWORD mode;
+
+ // GetConsoleMode() is an easy way to tell if the HANDLE is a
+ // console or not. We do not care about the value of mode since we
+ // do not plan to use any further Windows console functions.
+ return GetConsoleMode((HANDLE)handle, &mode);
+#else
+ return isatty(fd);
+#endif
+}
+
+
+extern bool
is_tty_stdin(void)
{
- const bool ret = isatty(STDIN_FILENO);
+ const bool ret = is_tty(STDIN_FILENO);
if (ret)
message_error(_("Compressed data cannot be read from "
@@ -277,7 +297,7 @@ is_tty_stdin(void)
extern bool
is_tty_stdout(void)
{
- const bool ret = isatty(STDOUT_FILENO);
+ const bool ret = is_tty(STDOUT_FILENO);
if (ret)
message_error(_("Compressed data cannot be written to "
diff --git a/src/xz/util.h b/src/xz/util.h
index 6d7e148..a2fdd05 100644
--- a/src/xz/util.h
+++ b/src/xz/util.h
@@ -1,3 +1,5 @@
+// SPDX-License-Identifier: 0BSD
+
///////////////////////////////////////////////////////////////////////////////
//
/// \file util.h
@@ -5,9 +7,6 @@
//
// Author: Lasse Collin
//
-// This file has been put into the public domain.
-// You can do whatever you want with this file.
-//
///////////////////////////////////////////////////////////////////////////////
/// \brief Safe malloc() that never returns NULL
@@ -105,6 +104,20 @@ lzma_attribute((__format__(__printf__, 3, 4)))
extern void my_snprintf(char **pos, size_t *left, const char *fmt, ...);
+/// \brief Test if file descriptor is a terminal
+///
+/// For POSIX systems, this is a simple wrapper around isatty(). However on
+/// Windows, isatty() returns true for all character devices, not just
+/// terminals.
+///
+/// \param fd File descriptor to test
+///
+/// \return bool:
+/// - true if file descriptor is a terminal
+/// - false otherwise
+extern bool is_tty(int fd);
+
+
/// \brief Test if stdin is a terminal
///
/// If stdin is a terminal, an error message is printed and exit status set
diff --git a/src/xz/xz.1 b/src/xz/xz.1
index 73ca6ef..5b880e8 100644
--- a/src/xz/xz.1
+++ b/src/xz/xz.1
@@ -1,12 +1,10 @@
'\" t
+.\" SPDX-License-Identifier: 0BSD
.\"
.\" Authors: Lasse Collin
.\" Jia Tan
.\"
-.\" This file has been put into the public domain.
-.\" You can do whatever you want with this file.
-.\"
-.TH XZ 1 "2023-07-17" "Tukaani" "XZ Utils"
+.TH XZ 1 "2024-04-08" "Tukaani" "XZ Utils"
.
.SH NAME
xz, unxz, xzcat, lzma, unlzma, lzcat \- Compress or decompress .xz and .lzma files
@@ -801,8 +799,6 @@ in the single-threaded mode.
It may vary slightly between
.B xz
versions.
-Memory requirements of some of the future multithreaded modes may
-be dramatically higher than that of the single-threaded mode.
.IP \(bu 3
DecMem contains the decompressor memory requirements.
That is, the compression settings determine
@@ -811,6 +807,15 @@ The exact decompressor memory usage is slightly more than
the LZMA2 dictionary size, but the values in the table
have been rounded up to the next full MiB.
.RE
+.IP ""
+Memory requirements of the multi-threaded mode are
+significantly higher than that of the single-threaded mode.
+With the default value of
+.BR \-\-block\-size ,
+each thread needs 3*3*DictSize plus CompMem or DecMem.
+For example, four threads with preset
+.B \-6
+needs 660\(en670\ MiB of memory.
.TP
.BR \-e ", " \-\-extreme
Use a slower variant of the selected compression preset level
@@ -902,50 +907,90 @@ Using
.I size
less than the LZMA2 dictionary size is waste of RAM
because then the LZMA2 dictionary buffer will never get fully used.
-The sizes of the blocks are stored in the block headers,
-which a future version of
-.B xz
-will use for multi-threaded decompression.
+In multi-threaded mode,
+the sizes of the blocks are stored in the block headers.
+This size information is required for multi-threaded decompression.
.IP ""
In single-threaded mode no block splitting is done by default.
Setting this option doesn't affect memory usage.
No size information is stored in block headers,
thus files created in single-threaded mode
won't be identical to files created in multi-threaded mode.
-The lack of size information also means that a future version of
+The lack of size information also means that
.B xz
won't be able decompress the files in multi-threaded mode.
.TP
-.BI \-\-block\-list= sizes
+.BI \-\-block\-list= items
When compressing to the
.B .xz
-format, start a new block after
+format, start a new block with an optional custom filter chain after
the given intervals of uncompressed data.
.IP ""
-The uncompressed
-.I sizes
-of the blocks are specified as a comma-separated list.
-Omitting a size (two or more consecutive commas) is a shorthand
-to use the size of the previous block.
+The
+.I items
+are a comma-separated list.
+Each item consists of an optional filter chain number
+between 0 and 9 followed by a colon
+.RB ( : )
+and a required size of uncompressed data.
+Omitting an item (two or more consecutive commas) is a
+shorthand to use the size and filters of the previous item.
.IP ""
If the input file is bigger than the sum of
-.IR sizes ,
-the last value in
-.I sizes
-is repeated until the end of the file.
+the sizes in
+.IR items ,
+the last item is repeated until the end of the file.
A special value of
.B 0
-may be used as the last value to indicate that
+may be used as the last size to indicate that
the rest of the file should be encoded as a single block.
.IP ""
-If one specifies
-.I sizes
-that exceed the encoder's block size
+An alternative filter chain for each block can be
+specified in combination with the
+.BI \-\-filters1= filters
+\&...\&
+.BI \-\-filters9= filters
+options.
+These options define filter chains with an identifier
+between 1\(en9.
+Filter chain 0 can be used to refer to the default filter chain,
+which is the same as not specifying a filter chain.
+The filter chain identifier can be used before the uncompressed
+size, followed by a colon
+.RB ( : ).
+For example, if one specifies
+.B \-\-block\-list=1:2MiB,3:2MiB,2:4MiB,,2MiB,0:4MiB
+then blocks will be created using:
+.RS
+.IP \(bu 3
+The filter chain specified by
+.B \-\-filters1
+and 2 MiB input
+.IP \(bu 3
+The filter chain specified by
+.B \-\-filters3
+and 2 MiB input
+.IP \(bu 3
+The filter chain specified by
+.B \-\-filters2
+and 4 MiB input
+.IP \(bu 3
+The filter chain specified by
+.B \-\-filters2
+and 4 MiB input
+.IP \(bu 3
+The default filter chain and 2 MiB input
+.IP \(bu 3
+The default filter chain and 4 MiB input for every block until
+end of input.
+.RE
+.IP ""
+If one specifies a size that exceeds the encoder's block size
(either the default value in threaded mode or
the value specified with \fB\-\-block\-size=\fIsize\fR),
the encoder will create additional blocks while
keeping the boundaries specified in
-.IR sizes .
+.IR items .
For example, if one specifies
.B \-\-block\-size=10MiB
.B \-\-block\-list=5MiB,10MiB,8MiB,12MiB,24MiB
@@ -1262,6 +1307,15 @@ meet this condition,
but files compressed in single-threaded mode don't even if
.BI \-\-block\-size= size
has been used.
+.IP ""
+The default value for
+.I threads
+is
+.BR 0 .
+In
+.B xz
+5.4.x and older the default is
+.BR 1 .
.
.SS "Custom compressor filter chains"
A custom filter chain allows specifying
@@ -1295,22 +1349,37 @@ in the chain.
Depending on the filter, this limitation is either inherent to
the filter design or exists to prevent security issues.
.PP
-A custom filter chain is specified by using one or more
-filter options in the order they are wanted in the filter chain.
-That is, the order of filter options is significant!
+A custom filter chain can be specified in two different ways.
+The options
+.BI \-\-filters= filters
+and
+.BI \-\-filters1= filters
+\&...\&
+.BI \-\-filters9= filters
+allow specifying an entire filter chain in one option using the
+liblzma filter string syntax.
+Alternatively, a filter chain can be specified by using one or more
+individual filter options in the order they are wanted in the filter chain.
+That is, the order of the individual filter options is significant!
When decoding raw streams
.RB ( \-\-format=raw ),
-the filter chain is specified in the same order as
+the filter chain must be specified in the same order as
it was specified when compressing.
-.PP
-Filters take filter-specific
+Any individual filter or preset options specified before the full
+chain option
+(\fB\-\-filters=\fIfilters\fR)
+will be forgotten.
+Individual filters specified after the full chain option will reset the
+filter chain.
+.PP
+Both the full and individual filter options take filter-specific
.I options
as a comma-separated list.
Extra commas in
.I options
are ignored.
-Every option has a default value, so you need to
-specify only those you want to change.
+Every option has a default value, so
+specify those you want to change.
.PP
To see the whole filter chain and
.IR options ,
@@ -1321,6 +1390,45 @@ use
twice).
This works also for viewing the filter chain options used by presets.
.TP
+.BI \-\-filters= filters
+Specify the full filter chain or a preset in a single option.
+Each filter can be separated by spaces or two dashes
+.RB ( \-\- ).
+.I filters
+may need to be quoted on the shell command line so it is
+parsed as a single option.
+To denote
+.IR options ,
+use
+.B :
+or
+.BR = .
+A preset can be prefixed with a
+.B \-
+and followed with zero or more flags.
+The only supported flag is
+.B e
+to apply the same options as
+.BR \-\-extreme .
+.TP
+\fB\-\-filters1\fR=\fIfilters\fR ... \fB\-\-filters9\fR=\fIfilters
+Specify up to nine additional filter chains that can be used with
+.BR \-\-block\-list .
+.IP ""
+For example, when compressing an archive with executable files
+followed by text files, the executable part could use a filter
+chain with a BCJ filter and the text part only the LZMA2 filter.
+.TP
+.B \-\-filters-help
+Display a help message describing how to specify presets and
+custom filter chains in the
+.B \-\-filters
+and
+.BI \-\-filters1= filters
+\&...\&
+.BI \-\-filters9= filters
+options, and exit successfully.
+.TP
\fB\-\-lzma1\fR[\fB=\fIoptions\fR]
.PD 0
.TP
@@ -1704,6 +1812,8 @@ and
\fB\-\-ia64\fR[\fB=\fIoptions\fR]
.TP
\fB\-\-sparc\fR[\fB=\fIoptions\fR]
+.TP
+\fB\-\-riscv\fR[\fB=\fIoptions\fR]
.PD
Add a branch/call/jump (BCJ) filter to the filter chain.
These filters can be used only as a non-last filter
@@ -1762,6 +1872,7 @@ ARM64;4;4096-byte alignment is best
PowerPC;4;Big endian only
IA-64;16;Itanium
SPARC;4;
+RISC-V;2;
.TE
.RE
.RE
@@ -1770,14 +1881,38 @@ Since the BCJ-filtered data is usually compressed with LZMA2,
the compression ratio may be improved slightly if
the LZMA2 options are set to match the
alignment of the selected BCJ filter.
-For example, with the IA-64 filter, it's good to set
-.B pb=4
-or even
+Examples:
+.RS
+.IP \(bu 3
+IA-64 filter has 16-byte alignment so
.B pb=4,lp=4,lc=0
+is good
with LZMA2 (2^4=16).
-The x86 filter is an exception;
-it's usually good to stick to LZMA2's default
-four-byte alignment when compressing x86 executables.
+.IP \(bu 3
+RISC-V code has 2-byte or 4-byte alignment
+depending on whether the file contains
+16-bit compressed instructions (the C extension).
+When 16-bit instructions are used,
+.B pb=2,lp=1,lc=3
+or
+.B pb=1,lp=1,lc=3
+is good.
+When 16-bit instructions aren't present,
+.B pb=2,lp=2,lc=2
+is the best.
+.B readelf \-h
+can be used to check if "RVC"
+appears on the "Flags" line.
+.IP \(bu 3
+ARM64 is always 4-byte aligned so
+.B pb=2,lp=2,lc=2
+is the best.
+.IP \(bu 3
+The x86 filter is an exception.
+It's usually good to stick to LZMA2's defaults
+.RB ( pb=2,lp=0,lc=3 )
+when compressing x86 executables.
+.RE
.IP ""
All BCJ filters support the same
.IR options :
@@ -1954,107 +2089,14 @@ easier to parse by other programs.
Currently
.B \-\-robot
is supported only together with
-.BR \-\-version ,
+.BR \-\-list ,
+.BR \-\-filters\-help ,
.BR \-\-info\-memory ,
and
-.BR \-\-list .
+.BR \-\-version .
It will be supported for compression and
decompression in the future.
.
-.SS Version
-.B "xz \-\-robot \-\-version"
-prints the version number of
-.B xz
-and liblzma in the following format:
-.PP
-.BI XZ_VERSION= XYYYZZZS
-.br
-.BI LIBLZMA_VERSION= XYYYZZZS
-.TP
-.I X
-Major version.
-.TP
-.I YYY
-Minor version.
-Even numbers are stable.
-Odd numbers are alpha or beta versions.
-.TP
-.I ZZZ
-Patch level for stable releases or
-just a counter for development releases.
-.TP
-.I S
-Stability.
-0 is alpha, 1 is beta, and 2 is stable.
-.I S
-should be always 2 when
-.I YYY
-is even.
-.PP
-.I XYYYZZZS
-are the same on both lines if
-.B xz
-and liblzma are from the same XZ Utils release.
-.PP
-Examples: 4.999.9beta is
-.B 49990091
-and
-5.0.0 is
-.BR 50000002 .
-.
-.SS "Memory limit information"
-.B "xz \-\-robot \-\-info\-memory"
-prints a single line with multiple tab-separated columns:
-.IP 1. 4
-Total amount of physical memory (RAM) in bytes.
-.IP 2. 4
-Memory usage limit for compression in bytes
-.RB ( \-\-memlimit\-compress ).
-A special value of
-.B 0
-indicates the default setting
-which for single-threaded mode is the same as no limit.
-.IP 3. 4
-Memory usage limit for decompression in bytes
-.RB ( \-\-memlimit\-decompress ).
-A special value of
-.B 0
-indicates the default setting
-which for single-threaded mode is the same as no limit.
-.IP 4. 4
-Since
-.B xz
-5.3.4alpha:
-Memory usage for multi-threaded decompression in bytes
-.RB ( \-\-memlimit\-mt\-decompress ).
-This is never zero because a system-specific default value
-shown in the column 5
-is used if no limit has been specified explicitly.
-This is also never greater than the value in the column 3
-even if a larger value has been specified with
-.BR \-\-memlimit\-mt\-decompress .
-.IP 5. 4
-Since
-.B xz
-5.3.4alpha:
-A system-specific default memory usage limit
-that is used to limit the number of threads
-when compressing with an automatic number of threads
-.RB ( \-\-threads=0 )
-and no memory usage limit has been specified
-.RB ( \-\-memlimit\-compress ).
-This is also used as the default value for
-.BR \-\-memlimit\-mt\-decompress .
-.IP 6. 4
-Since
-.B xz
-5.3.4alpha:
-Number of available processor threads.
-.PP
-In the future, the output of
-.B "xz \-\-robot \-\-info\-memory"
-may have more columns, but never more than a single line.
-.
.SS "List mode"
.B "xz \-\-robot \-\-list"
uses tab-separated output.
@@ -2339,6 +2381,127 @@ Future versions may add new line types and
new columns can be added to the existing line types,
but the existing columns won't be changed.
.
+.SS "Filters help"
+.B "xz \-\-robot \-\-filters-help"
+prints the supported filters in the following format:
+.PP
+\fIfilter\fB:\fIoption\fB=<\fIvalue\fB>,\fIoption\fB=<\fIvalue\fB>\fR...
+.TP
+.I filter
+Name of the filter
+.TP
+.I option
+Name of a filter specific option
+.TP
+.I value
+Numeric
+.I value
+ranges appear as
+\fB<\fImin\fB\-\fImax\fB>\fR.
+String
+.I value
+choices are shown within
+.B "< >"
+and separated by a
+.B |
+character.
+.PP
+Each filter is printed on its own line.
+.
+.SS "Memory limit information"
+.B "xz \-\-robot \-\-info\-memory"
+prints a single line with multiple tab-separated columns:
+.IP 1. 4
+Total amount of physical memory (RAM) in bytes.
+.IP 2. 4
+Memory usage limit for compression in bytes
+.RB ( \-\-memlimit\-compress ).
+A special value of
+.B 0
+indicates the default setting
+which for single-threaded mode is the same as no limit.
+.IP 3. 4
+Memory usage limit for decompression in bytes
+.RB ( \-\-memlimit\-decompress ).
+A special value of
+.B 0
+indicates the default setting
+which for single-threaded mode is the same as no limit.
+.IP 4. 4
+Since
+.B xz
+5.3.4alpha:
+Memory usage for multi-threaded decompression in bytes
+.RB ( \-\-memlimit\-mt\-decompress ).
+This is never zero because a system-specific default value
+shown in the column 5
+is used if no limit has been specified explicitly.
+This is also never greater than the value in the column 3
+even if a larger value has been specified with
+.BR \-\-memlimit\-mt\-decompress .
+.IP 5. 4
+Since
+.B xz
+5.3.4alpha:
+A system-specific default memory usage limit
+that is used to limit the number of threads
+when compressing with an automatic number of threads
+.RB ( \-\-threads=0 )
+and no memory usage limit has been specified
+.RB ( \-\-memlimit\-compress ).
+This is also used as the default value for
+.BR \-\-memlimit\-mt\-decompress .
+.IP 6. 4
+Since
+.B xz
+5.3.4alpha:
+Number of available processor threads.
+.PP
+In the future, the output of
+.B "xz \-\-robot \-\-info\-memory"
+may have more columns, but never more than a single line.
+.
+.SS Version
+.B "xz \-\-robot \-\-version"
+prints the version number of
+.B xz
+and liblzma in the following format:
+.PP
+.BI XZ_VERSION= XYYYZZZS
+.br
+.BI LIBLZMA_VERSION= XYYYZZZS
+.TP
+.I X
+Major version.
+.TP
+.I YYY
+Minor version.
+Even numbers are stable.
+Odd numbers are alpha or beta versions.
+.TP
+.I ZZZ
+Patch level for stable releases or
+just a counter for development releases.
+.TP
+.I S
+Stability.
+0 is alpha, 1 is beta, and 2 is stable.
+.I S
+should be always 2 when
+.I YYY
+is even.
+.PP
+.I XYYYZZZS
+are the same on both lines if
+.B xz
+and liblzma are from the same XZ Utils release.
+.PP
+Examples: 4.999.9beta is
+.B 49990091
+and
+5.0.0 is
+.BR 50000002 .
+.
.SH "EXIT STATUS"
.TP
.B 0
@@ -2391,7 +2554,7 @@ is run by a script or tool, for example, GNU
.RS
.PP
.nf
-.ft CW
+.ft CR
XZ_OPT=\-2v tar caf foo.tar.xz foo
.ft R
.fi
@@ -2411,7 +2574,7 @@ scripts one may use something like this:
.RS
.PP
.nf
-.ft CW
+.ft CR
XZ_OPT=${XZ_OPT\-"\-7e"}
export XZ_OPT
.ft R
@@ -2669,7 +2832,7 @@ if compression is successful:
.RS
.PP
.nf
-.ft CW
+.ft CR
xz foo
.ft R
.fi
@@ -2685,7 +2848,7 @@ even if decompression is successful:
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-dk bar.xz
.ft R
.fi
@@ -2703,7 +2866,7 @@ and 5\ MiB, respectively):
.RS
.PP
.nf
-.ft CW
+.ft CR
tar cf \- baz | xz \-4e > baz.tar.xz
.ft R
.fi
@@ -2714,7 +2877,7 @@ to standard output with a single command:
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-dcf a.txt b.txt.xz c.txt d.txt.lzma > abcd.txt
.ft R
.fi
@@ -2729,7 +2892,7 @@ can be used to parallelize compression of many files:
.RS
.PP
.nf
-.ft CW
+.ft CR
find . \-type f \e! \-name '*.xz' \-print0 \e
| xargs \-0r \-P4 \-n16 xz \-T1
.ft R
@@ -2769,7 +2932,7 @@ after compressing multiple files:
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-\-robot \-\-list *.xz | awk '/^totals/{print $5\-$4}'
.ft R
.fi
@@ -2789,7 +2952,7 @@ option:
.RS
.PP
.nf
-.ft CW
+.ft CR
if ! eval "$(xz \-\-robot \-\-version 2> /dev/null)" ||
[ "$XZ_VERSION" \-lt 50000002 ]; then
echo "Your xz is too old."
@@ -2805,7 +2968,7 @@ but if a limit has already been set, don't increase it:
.RS
.PP
.nf
-.ft CW
+.ft CR
NEWLIM=$((123 << 20))\ \ # 123 MiB
OLDLIM=$(xz \-\-robot \-\-info\-memory | cut \-f3)
if [ $OLDLIM \-eq 0 \-o $OLDLIM \-gt $NEWLIM ]; then
@@ -2858,7 +3021,7 @@ can be modified to use a bigger dictionary:
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-\-lzma2=preset=1,dict=32MiB foo.tar
.ft R
.fi
@@ -2886,7 +3049,7 @@ would use:
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-vv \-\-lzma2=dict=192MiB big_foo.tar
.ft R
.fi
@@ -2916,7 +3079,7 @@ using about 100\ KiB of memory.
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-\-check=crc32 \-\-lzma2=preset=6e,dict=64KiB foo
.ft R
.fi
@@ -2944,7 +3107,7 @@ slightly (like 0.1\ %) smaller file than
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-\-lzma2=preset=6e,pb=0,lc=4 source_code.tar
.ft R
.fi
@@ -2957,7 +3120,7 @@ using the x86 BCJ filter:
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-\-x86 \-\-lzma2 libfoo.so
.ft R
.fi
@@ -2992,7 +3155,7 @@ to LZMA2 to accommodate the three-byte alignment:
.RS
.PP
.nf
-.ft CW
+.ft CR
xz \-\-delta=dist=3 \-\-lzma2=pb=0 foo.tiff
.ft R
.fi
diff --git a/src/xz/xz_w32res.rc b/src/xz/xz_w32res.rc
index bad3020..e80903e 100644
--- a/src/xz/xz_w32res.rc
+++ b/src/xz/xz_w32res.rc
@@ -1,8 +1,7 @@
+/* SPDX-License-Identifier: 0BSD */
+
/*
* Author: Lasse Collin
- *
- * This file has been put into the public domain.
- * You can do whatever you want with this file.
*/
#define MY_TYPE VFT_APP
generated by cgit v1.2.3 (git 2.39.1) at 2024-11-22 22:10:33 +0000