diff options
Diffstat (limited to 'src/zstd/zlibWrapper')
| -rw-r--r-- | src/zstd/zlibWrapper/.gitignore | 28 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/BUCK | 22 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/Makefile | 117 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/README.md | 163 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/examples/example.c | 629 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/examples/example_original.c | 618 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/examples/fitblk.c | 254 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/examples/fitblk_original.c | 233 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/examples/minigzip.c | 640 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/examples/zwrapbench.c | 1001 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/gzclose.c | 28 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/gzcompatibility.h | 68 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/gzguts.h | 229 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/gzlib.c | 640 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/gzread.c | 678 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/gzwrite.c | 671 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/zstd_zlibwrapper.c | 1109 | ||||
| -rw-r--r-- | src/zstd/zlibWrapper/zstd_zlibwrapper.h | 88 |
18 files changed, 7216 insertions, 0 deletions
diff --git a/src/zstd/zlibWrapper/.gitignore b/src/zstd/zlibWrapper/.gitignore new file mode 100644 index 000000000..b037ae6f2 --- /dev/null +++ b/src/zstd/zlibWrapper/.gitignore @@ -0,0 +1,28 @@ +# object artifacts +*.o + +# Default result files +_* +example +example_zstd.* +example_gz.* +fitblk +fitblk_zstd.* +zwrapbench +foo.gz + +minigzip +minigzip_zstd +example +example_zstd +fitblk +fitblk_zstd +zwrapbench + +# Misc files +*.bat +*.zip +*.txt + +# Directories +minizip/ diff --git a/src/zstd/zlibWrapper/BUCK b/src/zstd/zlibWrapper/BUCK new file mode 100644 index 000000000..a0710e1cf --- /dev/null +++ b/src/zstd/zlibWrapper/BUCK @@ -0,0 +1,22 @@ +cxx_library( + name='zlib_wrapper', + visibility=['PUBLIC'], + exported_linker_flags=['-lz'], + header_namespace='', + exported_headers=['zstd_zlibwrapper.h'], + headers=[ + 'gzcompatibility.h', + 'gzguts.h', + ], + srcs=glob(['*.c']), + deps=[ + '//lib:zstd', + '//lib:zstd_common', + ], +) + +cxx_binary( + name='minigzip', + srcs=['examples/minigzip.c'], + deps=[':zlib_wrapper'], +) diff --git a/src/zstd/zlibWrapper/Makefile b/src/zstd/zlibWrapper/Makefile new file mode 100644 index 000000000..feed5b84a --- /dev/null +++ b/src/zstd/zlibWrapper/Makefile @@ -0,0 +1,117 @@ +# Makefile for example of using zstd wrapper for zlib +# +# make - compiles examples +# make MOREFLAGS=-DZWRAP_USE_ZSTD=1 - compiles examples with zstd compression turned on +# make test - runs examples + + +# Paths to static and dynamic zlib and zstd libraries +# Use "make ZLIB_PATH=path/to/zlib ZLIB_LIBRARY=path/to/libz.a" to select a path to library +ZLIB_LIBRARY ?= -lz +ZLIB_PATH ?= . + +ZSTDLIBDIR = ../lib +ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.a +ZLIBWRAPPER_PATH = . +GZFILES = gzclose.o gzlib.o gzread.o gzwrite.o +EXAMPLE_PATH = examples +PROGRAMS_PATH = ../programs +TEST_FILE = ../doc/zstd_compression_format.md + +CPPFLAGS += -DXXH_NAMESPACE=ZSTD_ -I$(ZLIB_PATH) -I$(PROGRAMS_PATH) \ + -I$(ZSTDLIBDIR) -I$(ZSTDLIBDIR)/common -I$(ZLIBWRAPPER_PATH) +STDFLAGS = -std=c89 -pedantic -Wno-long-long -Wno-variadic-macros -Wc++-compat \ + -DNO_snprintf -DNO_vsnprintf # strict ANSI C89 is missing these prototypes +DEBUGFLAGS= -Wall -Wextra -Wcast-qual -Wcast-align -Wshadow -Wswitch-enum \ + -Wdeclaration-after-statement -Wstrict-prototypes -Wundef \ + -Wstrict-aliasing=1 +CFLAGS ?= -O3 +CFLAGS += $(STDFLAGS) $(DEBUGFLAGS) $(MOREFLAGS) + +# Define *.exe as extension for Windows systems +ifneq (,$(filter Windows%,$(OS))) +EXT =.exe +else +EXT = +endif + +default : release + +release : STDFLAGS = +release : DEBUGFLAGS = +release : all + +all: fitblk example zwrapbench minigzip + +test: example fitblk example_zstd fitblk_zstd zwrapbench minigzip minigzip_zstd + ./example + ./example_zstd + ./fitblk 10240 <$(TEST_FILE) + ./fitblk 40960 <$(TEST_FILE) + ./fitblk_zstd 10240 <$(TEST_FILE) + ./fitblk_zstd 40960 <$(TEST_FILE) + @echo ---- minigzip start ---- + ./minigzip_zstd example$(EXT) + #cp example$(EXT).gz example$(EXT)_zstd.gz + ./minigzip_zstd -d example$(EXT).gz + ./minigzip example$(EXT) + #cp example$(EXT).gz example$(EXT)_gz.gz + ./minigzip_zstd -d example$(EXT).gz + @echo ---- minigzip end ---- + ./zwrapbench -qi1b3B1K $(TEST_FILE) + ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests + +#valgrindTest: ZSTDLIBRARY = $(ZSTDLIBDIR)/libzstd.so +valgrindTest: VALGRIND = LD_LIBRARY_PATH=$(ZSTDLIBDIR) valgrind --track-origins=yes --leak-check=full --error-exitcode=1 +valgrindTest: clean example fitblk example_zstd fitblk_zstd zwrapbench + @echo "\n ---- valgrind tests ----" + $(VALGRIND) ./example + $(VALGRIND) ./example_zstd + $(VALGRIND) ./fitblk 10240 <$(TEST_FILE) + $(VALGRIND) ./fitblk 40960 <$(TEST_FILE) + $(VALGRIND) ./fitblk_zstd 10240 <$(TEST_FILE) + $(VALGRIND) ./fitblk_zstd 40960 <$(TEST_FILE) + $(VALGRIND) ./zwrapbench -qi1b3B1K $(TEST_FILE) + $(VALGRIND) ./zwrapbench -rqi1b1e5 ../lib ../programs ../tests + +#.c.o: +# $(CC) $(CFLAGS) $(CPPFLAGS) -c $< -o $@ + +minigzip: $(EXAMPLE_PATH)/minigzip.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@ + +minigzip_zstd: $(EXAMPLE_PATH)/minigzip.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZSTDLIBRARY) $(ZLIB_LIBRARY) -o $@ + +example: $(EXAMPLE_PATH)/example.o zstd_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ + +example_zstd: $(EXAMPLE_PATH)/example.o zstdTurnedOn_zlibwrapper.o $(GZFILES) $(ZSTDLIBRARY) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ + +fitblk: $(EXAMPLE_PATH)/fitblk.o zstd_zlibwrapper.o $(ZSTDLIBRARY) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ + +fitblk_zstd: $(EXAMPLE_PATH)/fitblk.o zstdTurnedOn_zlibwrapper.o $(ZSTDLIBRARY) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ + +zwrapbench: $(EXAMPLE_PATH)/zwrapbench.o zstd_zlibwrapper.o $(PROGRAMS_PATH)/util.o $(PROGRAMS_PATH)/timefn.o $(PROGRAMS_PATH)/datagen.o $(ZSTDLIBRARY) + $(CC) $(CFLAGS) $(CPPFLAGS) $(LDFLAGS) $^ $(ZLIB_LIBRARY) -o $@ + + +zstd_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h + +zstdTurnedOn_zlibwrapper.o: CPPFLAGS += -DZWRAP_USE_ZSTD=1 +zstdTurnedOn_zlibwrapper.o: $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.c $(ZLIBWRAPPER_PATH)/zstd_zlibwrapper.h + $(CC) $(CPPFLAGS) $(CFLAGS) $< -c -o $@ + +$(ZSTDLIBDIR)/libzstd.a: + $(MAKE) -C $(ZSTDLIBDIR) libzstd.a + +$(ZSTDLIBDIR)/libzstd.so: + $(MAKE) -C $(ZSTDLIBDIR) libzstd + + +clean: + -$(RM) $(ZLIBWRAPPER_PATH)/*.o $(EXAMPLE_PATH)/*.o *.o foo.gz example$(EXT) example_zstd$(EXT) fitblk$(EXT) fitblk_zstd$(EXT) zwrapbench$(EXT) minigzip$(EXT) minigzip_zstd$(EXT) + @echo Cleaning completed diff --git a/src/zstd/zlibWrapper/README.md b/src/zstd/zlibWrapper/README.md new file mode 100644 index 000000000..e61767c46 --- /dev/null +++ b/src/zstd/zlibWrapper/README.md @@ -0,0 +1,163 @@ +Zstandard wrapper for zlib +================================ + +The main objective of creating a zstd wrapper for [zlib](http://zlib.net/) is to allow a quick and smooth transition to zstd for projects already using zlib. + +#### Required files + +To build the zstd wrapper for zlib the following files are required: +- zlib.h +- a static or dynamic zlib library +- zlibWrapper/zstd_zlibwrapper.h +- zlibWrapper/zstd_zlibwrapper.c +- zlibWrapper/gz*.c files (gzclose.c, gzlib.c, gzread.c, gzwrite.c) +- zlibWrapper/gz*.h files (gzcompatibility.h, gzguts.h) +- a static or dynamic zstd library + +The first two files are required by all projects using zlib and they are not included with the zstd distribution. +The further files are supplied with the zstd distribution. + + +#### Embedding the zstd wrapper within your project + +Let's assume that your project that uses zlib is compiled with: +```gcc project.o -lz``` + +To compile the zstd wrapper with your project you have to do the following: +- change all references with `#include "zlib.h"` to `#include "zstd_zlibwrapper.h"` +- compile your project with `zstd_zlibwrapper.c`, `gz*.c` and a static or dynamic zstd library + +The linking should be changed to: +```gcc project.o zstd_zlibwrapper.o gz*.c -lz -lzstd``` + + +#### Enabling zstd compression within your project + +After embedding the zstd wrapper within your project the zstd library is turned off by default. +Your project should work as before with zlib. There are two options to enable zstd compression: +- compilation with `-DZWRAP_USE_ZSTD=1` (or using `#define ZWRAP_USE_ZSTD 1` before `#include "zstd_zlibwrapper.h"`) +- using the `void ZWRAP_useZSTDcompression(int turn_on)` function (declared in `#include "zstd_zlibwrapper.h"`) + +During decompression zlib and zstd streams are automatically detected and decompressed using a proper library. +This behavior can be changed using `ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB)` what will make zlib decompression slightly faster. + + +#### Example +We have take the file `test/example.c` from [the zlib library distribution](http://zlib.net/) and copied it to [zlibWrapper/examples/example.c](examples/example.c). +After compilation and execution it shows the following results: +``` +zlib version 1.2.8 = 0x1280, compile flags = 0x65 +uncompress(): hello, hello! +gzread(): hello, hello! +gzgets() after gzseek: hello! +inflate(): hello, hello! +large_inflate(): OK +after inflateSync(): hello, hello! +inflate with dictionary: hello, hello! +``` +Then we have changed `#include "zlib.h"` to `#include "zstd_zlibwrapper.h"`, compiled the [example.c](examples/example.c) file +with `-DZWRAP_USE_ZSTD=1` and linked with additional `zstd_zlibwrapper.o gz*.c -lzstd`. +We were forced to turn off the following functions: `test_flush`, `test_sync` which use currently unsupported features. +After running it shows the following results: +``` +zlib version 1.2.8 = 0x1280, compile flags = 0x65 +uncompress(): hello, hello! +gzread(): hello, hello! +gzgets() after gzseek: hello! +inflate(): hello, hello! +large_inflate(): OK +inflate with dictionary: hello, hello! +``` +The script used for compilation can be found at [zlibWrapper/Makefile](Makefile). + + +#### The measurement of performance of Zstandard wrapper for zlib + +The zstd distribution contains a tool called `zwrapbench` which can measure speed and ratio of zlib, zstd, and the wrapper. +The benchmark is conducted using given filenames or synthetic data if filenames are not provided. +The files are read into memory and processed independently. +It makes benchmark more precise as it eliminates I/O overhead. +Many filenames can be supplied as multiple parameters, parameters with wildcards or names of directories can be used as parameters with the -r option. +One can select compression levels starting from `-b` and ending with `-e`. The `-i` parameter selects minimal time used for each of tested levels. +With `-B` option bigger files can be divided into smaller, independently compressed blocks. +The benchmark tool can be compiled with `make zwrapbench` using [zlibWrapper/Makefile](Makefile). + + +#### Improving speed of streaming compression + +During streaming compression the compressor never knows how big is data to compress. +Zstandard compression can be improved by providing size of source data to the compressor. By default streaming compressor assumes that data is bigger than 256 KB but it can hurt compression speed on smaller data. +The zstd wrapper provides the `ZWRAP_setPledgedSrcSize()` function that allows to change a pledged source size for a given compression stream. +The function will change zstd compression parameters what may improve compression speed and/or ratio. +It should be called just after `deflateInit()`or `deflateReset()` and before `deflate()` or `deflateSetDictionary()`. The function is only helpful when data is compressed in blocks. There will be no change in case of `deflateInit()` or `deflateReset()` immediately followed by `deflate(strm, Z_FINISH)` +as this case is automatically detected. + + +#### Reusing contexts + +The ordinary zlib compression of two files/streams allocates two contexts: +- for the 1st file calls `deflateInit`, `deflate`, `...`, `deflate`, `deflateEnd` +- for the 2nd file calls `deflateInit`, `deflate`, `...`, `deflate`, `deflateEnd` + +The speed of compression can be improved with reusing a single context with following steps: +- initialize the context with `deflateInit` +- for the 1st file call `deflate`, `...`, `deflate` +- for the 2nd file call `deflateReset`, `deflate`, `...`, `deflate` +- free the context with `deflateEnd` + +To check the difference we made experiments using `zwrapbench -ri6b6` with zstd and zlib compression (both at level 6). +The input data was decompressed git repository downloaded from https://github.com/git/git/archive/master.zip which contains 2979 files. +The table below shows that reusing contexts has a minor influence on zlib but it gives improvement for zstd. +In our example (the last 2 lines) it gives 4% better compression speed and 5% better decompression speed. + +| Compression type | Compression | Decompress.| Compr. size | Ratio | +| ------------------------------------------------- | ------------| -----------| ----------- | ----- | +| zlib 1.2.8 | 30.51 MB/s | 219.3 MB/s | 6819783 | 3.459 | +| zlib 1.2.8 not reusing a context | 30.22 MB/s | 218.1 MB/s | 6819783 | 3.459 | +| zlib 1.2.8 with zlibWrapper and reusing a context | 30.40 MB/s | 218.9 MB/s | 6819783 | 3.459 | +| zlib 1.2.8 with zlibWrapper not reusing a context | 30.28 MB/s | 218.1 MB/s | 6819783 | 3.459 | +| zstd 1.1.0 using ZSTD_CCtx | 68.35 MB/s | 430.9 MB/s | 6868521 | 3.435 | +| zstd 1.1.0 using ZSTD_CStream | 66.63 MB/s | 422.3 MB/s | 6868521 | 3.435 | +| zstd 1.1.0 with zlibWrapper and reusing a context | 54.01 MB/s | 403.2 MB/s | 6763482 | 3.488 | +| zstd 1.1.0 with zlibWrapper not reusing a context | 51.59 MB/s | 383.7 MB/s | 6763482 | 3.488 | + + +#### Compatibility issues +After enabling zstd compression not all native zlib functions are supported. When calling unsupported methods they put error message into `strm->msg` and return Z_STREAM_ERROR. + +Supported methods: +- deflateInit +- deflate (with exception of Z_FULL_FLUSH, Z_BLOCK, and Z_TREES) +- deflateSetDictionary +- deflateEnd +- deflateReset +- deflateBound +- inflateInit +- inflate +- inflateSetDictionary +- inflateReset +- inflateReset2 +- compress +- compress2 +- compressBound +- uncompress +- gzip file access functions + +Ignored methods (they do nothing): +- deflateParams + +Unsupported methods: +- deflateCopy +- deflateTune +- deflatePending +- deflatePrime +- deflateSetHeader +- inflateGetDictionary +- inflateCopy +- inflateSync +- inflatePrime +- inflateMark +- inflateGetHeader +- inflateBackInit +- inflateBack +- inflateBackEnd diff --git a/src/zstd/zlibWrapper/examples/example.c b/src/zstd/zlibWrapper/examples/example.c new file mode 100644 index 000000000..9000f7a32 --- /dev/null +++ b/src/zstd/zlibWrapper/examples/example.c @@ -0,0 +1,629 @@ +/* example.c contains minimal changes required to be compiled with zlibWrapper: + * - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h" + * - test_flush() and test_sync() use functions not supported by zlibWrapper + therefore they are disabled while zstd compression is turned on */ + +/* example.c -- usage example of the zlib compression library + */ +/* + Copyright (c) 1995-2006, 2011 Jean-loup Gailly + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgement in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + */ + +/* @(#) $Id$ */ + +#include "zstd_zlibwrapper.h" +#include <stdio.h> + +#ifdef STDC +# include <string.h> +# include <stdlib.h> +#endif + +#if defined(VMS) || defined(RISCOS) +# define TESTFILE "foo-gz" +#else +# define TESTFILE "foo.gz" +#endif + +#define CHECK_ERR(err, msg) { \ + if (err != Z_OK) { \ + fprintf(stderr, "%s error: %d\n", msg, err); \ + exit(1); \ + } \ +} + +z_const char hello[] = "hello, hello! I said hello, hello!"; +/* "hello world" would be more standard, but the repeated "hello" + * stresses the compression code better, sorry... + */ + +const char dictionary[] = "hello, hello!"; +uLong dictId; /* Adler32 value of the dictionary */ + +void test_deflate OF((Byte *compr, uLong comprLen)); +void test_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_large_deflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_large_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_flush OF((Byte *compr, uLong *comprLen)); +void test_sync OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_dict_deflate OF((Byte *compr, uLong comprLen)); +void test_dict_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +int main OF((int argc, char *argv[])); + + +#ifdef Z_SOLO + +void *myalloc OF((void *, unsigned, unsigned)); +void myfree OF((void *, void *)); + +void *myalloc(q, n, m) + void *q; + unsigned n, m; +{ + void *buf = calloc(n, m); + q = Z_NULL; + /* printf("myalloc %p n=%d m=%d\n", buf, n, m); */ + return buf; +} + +void myfree(void *q, void *p) +{ + /* printf("myfree %p\n", p); */ + q = Z_NULL; + free(p); +} + +static alloc_func zalloc = myalloc; +static free_func zfree = myfree; + +#else /* !Z_SOLO */ + +static alloc_func zalloc = (alloc_func)0; +static free_func zfree = (free_func)0; + +void test_compress OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_gzio OF((const char *fname, + Byte *uncompr, uLong uncomprLen)); + +/* =========================================================================== + * Test compress() and uncompress() + */ +void test_compress(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + uLong len = (uLong)strlen(hello)+1; + + err = compress(compr, &comprLen, (const Bytef*)hello, len); + CHECK_ERR(err, "compress"); + + strcpy((char*)uncompr, "garbage"); + + err = uncompress(uncompr, &uncomprLen, compr, comprLen); + CHECK_ERR(err, "uncompress"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad uncompress\n"); + exit(1); + } else { + printf("uncompress(): %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Test read/write of .gz files + */ +void test_gzio(fname, uncompr, uncomprLen) + const char *fname; /* compressed file name */ + Byte *uncompr; + uLong uncomprLen; +{ +#ifdef NO_GZCOMPRESS + fprintf(stderr, "NO_GZCOMPRESS -- gz* functions cannot compress\n"); +#else + int err; + int len = (int)strlen(hello)+1; + gzFile file; + z_off_t pos; + + file = gzopen(fname, "wb"); + if (file == NULL) { + fprintf(stderr, "gzopen error\n"); + exit(1); + } + gzputc(file, 'h'); + if (gzputs(file, "ello") != 4) { + fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err)); + exit(1); + } + if (gzprintf(file, ", %s! I said hello, hello!", "hello") != 8+21) { + fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err)); + exit(1); + } + gzseek(file, 1L, SEEK_CUR); /* add one zero byte */ + gzclose(file); + + file = gzopen(fname, "rb"); + if (file == NULL) { + fprintf(stderr, "gzopen error\n"); + exit(1); + } + strcpy((char*)uncompr, "garbage"); + + if (gzread(file, uncompr, (unsigned)uncomprLen) != len) { + fprintf(stderr, "gzread err: %s\n", gzerror(file, &err)); + exit(1); + } + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad gzread: %s\n", (char*)uncompr); + exit(1); + } else { + printf("gzread(): %s\n", (char*)uncompr); + } + + pos = gzseek(file, -8L, SEEK_CUR); + if (pos != 6+21 || gztell(file) != pos) { + fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n", + (long)pos, (long)gztell(file)); + exit(1); + } + + if (gzgetc(file) != ' ') { + fprintf(stderr, "gzgetc error\n"); + exit(1); + } + + if (gzungetc(' ', file) != ' ') { + fprintf(stderr, "gzungetc error\n"); + exit(1); + } + + gzgets(file, (char*)uncompr, (int)uncomprLen); + if (strlen((char*)uncompr) != 7) { /* " hello!" */ + fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err)); + exit(1); + } + if (strcmp((char*)uncompr, hello + 6+21)) { + fprintf(stderr, "bad gzgets after gzseek\n"); + exit(1); + } else { + printf("gzgets() after gzseek: %s\n", (char*)uncompr); + } + + gzclose(file); +#endif +} + +#endif /* Z_SOLO */ + +/* =========================================================================== + * Test deflate() with small buffers + */ +void test_deflate(compr, comprLen) + Byte *compr; + uLong comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + uLong len = (uLong)strlen(hello)+1; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.next_out = compr; + + while (c_stream.total_in != len && c_stream.total_out < comprLen) { + c_stream.avail_in = c_stream.avail_out = 1; /* force small buffers */ + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + } + /* Finish the stream, still forcing small buffers: */ + for (;;) { + c_stream.avail_out = 1; + err = deflate(&c_stream, Z_FINISH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "deflate"); + } + + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with small buffers + */ +void test_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = 0; + d_stream.next_out = uncompr; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + while (d_stream.total_out < uncomprLen && d_stream.total_in < comprLen) { + d_stream.avail_in = d_stream.avail_out = 1; /* force small buffers */ + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "inflate"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad inflate\n"); + exit(1); + } else { + printf("inflate(): %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Test deflate() with large buffers and dynamic change of compression level + */ +void test_large_deflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_BEST_SPEED); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_out = compr; + c_stream.avail_out = (uInt)comprLen; + + /* At this point, uncompr is still mostly zeroes, so it should compress + * very well: + */ + c_stream.next_in = uncompr; + c_stream.avail_in = (uInt)uncomprLen; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + if (c_stream.avail_in != 0) { + fprintf(stderr, "deflate not greedy\n"); + exit(1); + } + + /* Feed in already compressed data and switch to no compression: */ + deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY); + c_stream.next_in = compr; + c_stream.avail_in = (uInt)comprLen/2; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + + /* Switch back to compressing mode: */ + deflateParams(&c_stream, Z_BEST_COMPRESSION, Z_FILTERED); + c_stream.next_in = uncompr; + c_stream.avail_in = (uInt)uncomprLen; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + fprintf(stderr, "deflate should report Z_STREAM_END\n"); + exit(1); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with large buffers + */ +void test_large_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = (uInt)comprLen; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + for (;;) { + d_stream.next_out = uncompr; /* discard the output */ + d_stream.avail_out = (uInt)uncomprLen; + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "large inflate"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (d_stream.total_out != 2*uncomprLen + comprLen/2) { + fprintf(stderr, "bad large inflate: %ld\n", d_stream.total_out); + exit(1); + } else { + printf("large_inflate(): OK\n"); + } +} + +/* =========================================================================== + * Test deflate() with full flush + */ +void test_flush(compr, comprLen) + Byte *compr; + uLong *comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + uInt len = (uInt)strlen(hello)+1; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.next_out = compr; + c_stream.avail_in = 3; + c_stream.avail_out = (uInt)*comprLen; + err = deflate(&c_stream, Z_FULL_FLUSH); + CHECK_ERR(err, "deflate"); + + compr[3]++; /* force an error in first compressed block */ + c_stream.avail_in = len - 3; + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + CHECK_ERR(err, "deflate"); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); + + *comprLen = c_stream.total_out; +} + +/* =========================================================================== + * Test inflateSync() + */ +void test_sync(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = 2; /* just read the zlib header */ + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + d_stream.next_out = uncompr; + d_stream.avail_out = (uInt)uncomprLen; + + inflate(&d_stream, Z_NO_FLUSH); + CHECK_ERR(err, "inflate"); + + d_stream.avail_in = (uInt)comprLen-2; /* read all compressed data */ + err = inflateSync(&d_stream); /* but skip the damaged part */ + CHECK_ERR(err, "inflateSync"); + + err = inflate(&d_stream, Z_FINISH); + if (err != Z_DATA_ERROR) { + fprintf(stderr, "inflate should report DATA_ERROR\n"); + /* Because of incorrect adler32 */ + exit(1); + } + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + printf("after inflateSync(): hel%s\n", (char *)uncompr); +} + +/* =========================================================================== + * Test deflate() with preset dictionary + */ +void test_dict_deflate(compr, comprLen) + Byte *compr; + uLong comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_BEST_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + err = deflateSetDictionary(&c_stream, + (const Bytef*)dictionary, (int)sizeof(dictionary)); + CHECK_ERR(err, "deflateSetDictionary"); + + dictId = c_stream.adler; + c_stream.next_out = compr; + c_stream.avail_out = (uInt)comprLen; + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.avail_in = (uInt)strlen(hello)+1; + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + fprintf(stderr, "deflate should report Z_STREAM_END\n"); + exit(1); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with a preset dictionary + */ +void test_dict_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = (uInt)comprLen; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + d_stream.next_out = uncompr; + d_stream.avail_out = (uInt)uncomprLen; + + for (;;) { + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + if (err == Z_NEED_DICT) { + if (d_stream.adler != dictId) { + fprintf(stderr, "unexpected dictionary"); + exit(1); + } + err = inflateSetDictionary(&d_stream, (const Bytef*)dictionary, + (int)sizeof(dictionary)); + } + CHECK_ERR(err, "inflate with dict"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad inflate with dict\n"); + exit(1); + } else { + printf("inflate with dictionary: %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Usage: example [output.gz [input.gz]] + */ + +int main(argc, argv) + int argc; + char *argv[]; +{ + Byte *compr, *uncompr; + uLong comprLen = 10000*sizeof(int); /* don't overflow on MSDOS */ + uLong uncomprLen = comprLen; + static const char* myVersion = ZLIB_VERSION; + + if (zlibVersion()[0] != myVersion[0]) { + fprintf(stderr, "incompatible zlib version\n"); + exit(1); + + } else if (strcmp(zlibVersion(), ZLIB_VERSION) != 0) { + fprintf(stderr, "warning: different zlib version\n"); + } + + printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", + ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); + if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); + + compr = (Byte*)calloc((uInt)comprLen, 1); + uncompr = (Byte*)calloc((uInt)uncomprLen, 1); + /* compr and uncompr are cleared to avoid reading uninitialized + * data and to ensure that uncompr compresses well. + */ + if (compr == Z_NULL || uncompr == Z_NULL) { + printf("out of memory\n"); + exit(1); + } + +#ifdef Z_SOLO + argc = strlen(argv[0]); +#else + test_compress(compr, comprLen, uncompr, uncomprLen); + + test_gzio((argc > 1 ? argv[1] : TESTFILE), + uncompr, uncomprLen); +#endif + + test_deflate(compr, comprLen); + test_inflate(compr, comprLen, uncompr, uncomprLen); + + test_large_deflate(compr, comprLen, uncompr, uncomprLen); + test_large_inflate(compr, comprLen, uncompr, uncomprLen); + + if (!ZWRAP_isUsingZSTDcompression()) { + test_flush(compr, &comprLen); + test_sync(compr, comprLen, uncompr, uncomprLen); + } + comprLen = uncomprLen; + + test_dict_deflate(compr, comprLen); + test_dict_inflate(compr, comprLen, uncompr, uncomprLen); + + free(compr); + free(uncompr); + + return 0; +} diff --git a/src/zstd/zlibWrapper/examples/example_original.c b/src/zstd/zlibWrapper/examples/example_original.c new file mode 100644 index 000000000..649882bf0 --- /dev/null +++ b/src/zstd/zlibWrapper/examples/example_original.c @@ -0,0 +1,618 @@ +/* example.c -- usage example of the zlib compression library + */ +/* + Copyright (c) 1995-2006, 2011 Jean-loup Gailly + +This software is provided 'as-is', without any express or implied +warranty. In no event will the authors be held liable for any damages +arising from the use of this software. + +Permission is granted to anyone to use this software for any purpose, +including commercial applications, and to alter it and redistribute it +freely, subject to the following restrictions: + +1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgement in the product documentation would be + appreciated but is not required. +2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. +3. This notice may not be removed or altered from any source distribution. +*/ + +/* @(#) $Id$ */ + +#include "zlib.h" +#include <stdio.h> + +#ifdef STDC +# include <string.h> +# include <stdlib.h> +#endif + +#if defined(VMS) || defined(RISCOS) +# define TESTFILE "foo-gz" +#else +# define TESTFILE "foo.gz" +#endif + +#define CHECK_ERR(err, msg) { \ + if (err != Z_OK) { \ + fprintf(stderr, "%s error: %d\n", msg, err); \ + exit(1); \ + } \ +} + +z_const char hello[] = "hello, hello!"; +/* "hello world" would be more standard, but the repeated "hello" + * stresses the compression code better, sorry... + */ + +const char dictionary[] = "hello"; +uLong dictId; /* Adler32 value of the dictionary */ + +void test_deflate OF((Byte *compr, uLong comprLen)); +void test_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_large_deflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_large_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_flush OF((Byte *compr, uLong *comprLen)); +void test_sync OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_dict_deflate OF((Byte *compr, uLong comprLen)); +void test_dict_inflate OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +int main OF((int argc, char *argv[])); + + +#ifdef Z_SOLO + +void *myalloc OF((void *, unsigned, unsigned)); +void myfree OF((void *, void *)); + +void *myalloc(q, n, m) + void *q; + unsigned n, m; +{ + q = Z_NULL; + return calloc(n, m); +} + +void myfree(void *q, void *p) +{ + q = Z_NULL; + free(p); +} + +static alloc_func zalloc = myalloc; +static free_func zfree = myfree; + +#else /* !Z_SOLO */ + +static alloc_func zalloc = (alloc_func)0; +static free_func zfree = (free_func)0; + +void test_compress OF((Byte *compr, uLong comprLen, + Byte *uncompr, uLong uncomprLen)); +void test_gzio OF((const char *fname, + Byte *uncompr, uLong uncomprLen)); + +/* =========================================================================== + * Test compress() and uncompress() + */ +void test_compress(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + uLong len = (uLong)strlen(hello)+1; + + err = compress(compr, &comprLen, (const Bytef*)hello, len); + CHECK_ERR(err, "compress"); + + strcpy((char*)uncompr, "garbage"); + + err = uncompress(uncompr, &uncomprLen, compr, comprLen); + CHECK_ERR(err, "uncompress"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad uncompress\n"); + exit(1); + } else { + printf("uncompress(): %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Test read/write of .gz files + */ +void test_gzio(fname, uncompr, uncomprLen) + const char *fname; /* compressed file name */ + Byte *uncompr; + uLong uncomprLen; +{ +#ifdef NO_GZCOMPRESS + fprintf(stderr, "NO_GZCOMPRESS -- gz* functions cannot compress\n"); +#else + int err; + int len = (int)strlen(hello)+1; + gzFile file; + z_off_t pos; + + file = gzopen(fname, "wb"); + if (file == NULL) { + fprintf(stderr, "gzopen error\n"); + exit(1); + } + gzputc(file, 'h'); + if (gzputs(file, "ello") != 4) { + fprintf(stderr, "gzputs err: %s\n", gzerror(file, &err)); + exit(1); + } + if (gzprintf(file, ", %s!", "hello") != 8) { + fprintf(stderr, "gzprintf err: %s\n", gzerror(file, &err)); + exit(1); + } + gzseek(file, 1L, SEEK_CUR); /* add one zero byte */ + gzclose(file); + + file = gzopen(fname, "rb"); + if (file == NULL) { + fprintf(stderr, "gzopen error\n"); + exit(1); + } + strcpy((char*)uncompr, "garbage"); + + if (gzread(file, uncompr, (unsigned)uncomprLen) != len) { + fprintf(stderr, "gzread err: %s\n", gzerror(file, &err)); + exit(1); + } + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad gzread: %s\n", (char*)uncompr); + exit(1); + } else { + printf("gzread(): %s\n", (char*)uncompr); + } + + pos = gzseek(file, -8L, SEEK_CUR); + if (pos != 6 || gztell(file) != pos) { + fprintf(stderr, "gzseek error, pos=%ld, gztell=%ld\n", + (long)pos, (long)gztell(file)); + exit(1); + } + + if (gzgetc(file) != ' ') { + fprintf(stderr, "gzgetc error\n"); + exit(1); + } + + if (gzungetc(' ', file) != ' ') { + fprintf(stderr, "gzungetc error\n"); + exit(1); + } + + gzgets(file, (char*)uncompr, (int)uncomprLen); + if (strlen((char*)uncompr) != 7) { /* " hello!" */ + fprintf(stderr, "gzgets err after gzseek: %s\n", gzerror(file, &err)); + exit(1); + } + if (strcmp((char*)uncompr, hello + 6)) { + fprintf(stderr, "bad gzgets after gzseek\n"); + exit(1); + } else { + printf("gzgets() after gzseek: %s\n", (char*)uncompr); + } + + gzclose(file); +#endif +} + +#endif /* Z_SOLO */ + +/* =========================================================================== + * Test deflate() with small buffers + */ +void test_deflate(compr, comprLen) + Byte *compr; + uLong comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + uLong len = (uLong)strlen(hello)+1; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.next_out = compr; + + while (c_stream.total_in != len && c_stream.total_out < comprLen) { + c_stream.avail_in = c_stream.avail_out = 1; /* force small buffers */ + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + } + /* Finish the stream, still forcing small buffers: */ + for (;;) { + c_stream.avail_out = 1; + err = deflate(&c_stream, Z_FINISH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "deflate"); + } + + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with small buffers + */ +void test_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = 0; + d_stream.next_out = uncompr; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + while (d_stream.total_out < uncomprLen && d_stream.total_in < comprLen) { + d_stream.avail_in = d_stream.avail_out = 1; /* force small buffers */ + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "inflate"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad inflate\n"); + exit(1); + } else { + printf("inflate(): %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Test deflate() with large buffers and dynamic change of compression level + */ +void test_large_deflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_BEST_SPEED); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_out = compr; + c_stream.avail_out = (uInt)comprLen; + + /* At this point, uncompr is still mostly zeroes, so it should compress + * very well: + */ + c_stream.next_in = uncompr; + c_stream.avail_in = (uInt)uncomprLen; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + if (c_stream.avail_in != 0) { + fprintf(stderr, "deflate not greedy\n"); + exit(1); + } + + /* Feed in already compressed data and switch to no compression: */ + deflateParams(&c_stream, Z_NO_COMPRESSION, Z_DEFAULT_STRATEGY); + c_stream.next_in = compr; + c_stream.avail_in = (uInt)comprLen/2; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + + /* Switch back to compressing mode: */ + deflateParams(&c_stream, Z_BEST_COMPRESSION, Z_FILTERED); + c_stream.next_in = uncompr; + c_stream.avail_in = (uInt)uncomprLen; + err = deflate(&c_stream, Z_NO_FLUSH); + CHECK_ERR(err, "deflate"); + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + fprintf(stderr, "deflate should report Z_STREAM_END\n"); + exit(1); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with large buffers + */ +void test_large_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = (uInt)comprLen; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + for (;;) { + d_stream.next_out = uncompr; /* discard the output */ + d_stream.avail_out = (uInt)uncomprLen; + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + CHECK_ERR(err, "large inflate"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (d_stream.total_out != 2*uncomprLen + comprLen/2) { + fprintf(stderr, "bad large inflate: %ld\n", d_stream.total_out); + exit(1); + } else { + printf("large_inflate(): OK\n"); + } +} + +/* =========================================================================== + * Test deflate() with full flush + */ +void test_flush(compr, comprLen) + Byte *compr; + uLong *comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + uInt len = (uInt)strlen(hello)+1; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_DEFAULT_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.next_out = compr; + c_stream.avail_in = 3; + c_stream.avail_out = (uInt)*comprLen; + err = deflate(&c_stream, Z_FULL_FLUSH); + CHECK_ERR(err, "deflate"); + + compr[3]++; /* force an error in first compressed block */ + c_stream.avail_in = len - 3; + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + CHECK_ERR(err, "deflate"); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); + + *comprLen = c_stream.total_out; +} + +/* =========================================================================== + * Test inflateSync() + */ +void test_sync(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = 2; /* just read the zlib header */ + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + d_stream.next_out = uncompr; + d_stream.avail_out = (uInt)uncomprLen; + + inflate(&d_stream, Z_NO_FLUSH); + CHECK_ERR(err, "inflate"); + + d_stream.avail_in = (uInt)comprLen-2; /* read all compressed data */ + err = inflateSync(&d_stream); /* but skip the damaged part */ + CHECK_ERR(err, "inflateSync"); + + err = inflate(&d_stream, Z_FINISH); + if (err != Z_DATA_ERROR) { + fprintf(stderr, "inflate should report DATA_ERROR\n"); + /* Because of incorrect adler32 */ + exit(1); + } + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + printf("after inflateSync(): hel%s\n", (char *)uncompr); +} + +/* =========================================================================== + * Test deflate() with preset dictionary + */ +void test_dict_deflate(compr, comprLen) + Byte *compr; + uLong comprLen; +{ + z_stream c_stream; /* compression stream */ + int err; + + c_stream.zalloc = zalloc; + c_stream.zfree = zfree; + c_stream.opaque = (voidpf)0; + + err = deflateInit(&c_stream, Z_BEST_COMPRESSION); + CHECK_ERR(err, "deflateInit"); + + err = deflateSetDictionary(&c_stream, + (const Bytef*)dictionary, (int)sizeof(dictionary)); + CHECK_ERR(err, "deflateSetDictionary"); + + dictId = c_stream.adler; + c_stream.next_out = compr; + c_stream.avail_out = (uInt)comprLen; + + c_stream.next_in = (z_const unsigned char *)hello; + c_stream.avail_in = (uInt)strlen(hello)+1; + + err = deflate(&c_stream, Z_FINISH); + if (err != Z_STREAM_END) { + fprintf(stderr, "deflate should report Z_STREAM_END\n"); + exit(1); + } + err = deflateEnd(&c_stream); + CHECK_ERR(err, "deflateEnd"); +} + +/* =========================================================================== + * Test inflate() with a preset dictionary + */ +void test_dict_inflate(compr, comprLen, uncompr, uncomprLen) + Byte *compr, *uncompr; + uLong comprLen, uncomprLen; +{ + int err; + z_stream d_stream; /* decompression stream */ + + strcpy((char*)uncompr, "garbage"); + + d_stream.zalloc = zalloc; + d_stream.zfree = zfree; + d_stream.opaque = (voidpf)0; + + d_stream.next_in = compr; + d_stream.avail_in = (uInt)comprLen; + + err = inflateInit(&d_stream); + CHECK_ERR(err, "inflateInit"); + + d_stream.next_out = uncompr; + d_stream.avail_out = (uInt)uncomprLen; + + for (;;) { + err = inflate(&d_stream, Z_NO_FLUSH); + if (err == Z_STREAM_END) break; + if (err == Z_NEED_DICT) { + if (d_stream.adler != dictId) { + fprintf(stderr, "unexpected dictionary"); + exit(1); + } + err = inflateSetDictionary(&d_stream, (const Bytef*)dictionary, + (int)sizeof(dictionary)); + } + CHECK_ERR(err, "inflate with dict"); + } + + err = inflateEnd(&d_stream); + CHECK_ERR(err, "inflateEnd"); + + if (strcmp((char*)uncompr, hello)) { + fprintf(stderr, "bad inflate with dict\n"); + exit(1); + } else { + printf("inflate with dictionary: %s\n", (char *)uncompr); + } +} + +/* =========================================================================== + * Usage: example [output.gz [input.gz]] + */ + +int main(argc, argv) + int argc; + char *argv[]; +{ + Byte *compr, *uncompr; + uLong comprLen = 10000*sizeof(int); /* don't overflow on MSDOS */ + uLong uncomprLen = comprLen; + static const char* myVersion = ZLIB_VERSION; + + if (zlibVersion()[0] != myVersion[0]) { + fprintf(stderr, "incompatible zlib version\n"); + exit(1); + + } else if (strcmp(zlibVersion(), ZLIB_VERSION) != 0) { + fprintf(stderr, "warning: different zlib version\n"); + } + + printf("zlib version %s = 0x%04x, compile flags = 0x%lx\n", + ZLIB_VERSION, ZLIB_VERNUM, zlibCompileFlags()); + + compr = (Byte*)calloc((uInt)comprLen, 1); + uncompr = (Byte*)calloc((uInt)uncomprLen, 1); + /* compr and uncompr are cleared to avoid reading uninitialized + * data and to ensure that uncompr compresses well. + */ + if (compr == Z_NULL || uncompr == Z_NULL) { + printf("out of memory\n"); + exit(1); + } + +#ifdef Z_SOLO + argc = strlen(argv[0]); +#else + test_compress(compr, comprLen, uncompr, uncomprLen); + + test_gzio((argc > 1 ? argv[1] : TESTFILE), + uncompr, uncomprLen); +#endif + + test_deflate(compr, comprLen); + test_inflate(compr, comprLen, uncompr, uncomprLen); + + test_large_deflate(compr, comprLen, uncompr, uncomprLen); + test_large_inflate(compr, comprLen, uncompr, uncomprLen); + + test_flush(compr, &comprLen); + test_sync(compr, comprLen, uncompr, uncomprLen); + comprLen = uncomprLen; + + test_dict_deflate(compr, comprLen); + test_dict_inflate(compr, comprLen, uncompr, uncomprLen); + + free(compr); + free(uncompr); + + return 0; +} diff --git a/src/zstd/zlibWrapper/examples/fitblk.c b/src/zstd/zlibWrapper/examples/fitblk.c new file mode 100644 index 000000000..669b176eb --- /dev/null +++ b/src/zstd/zlibWrapper/examples/fitblk.c @@ -0,0 +1,254 @@ +/* fitblk.c contains minimal changes required to be compiled with zlibWrapper: + * - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h" + * - writing block to stdout was disabled */ + +/* fitblk.c: example of fitting compressed output to a specified size + Not copyrighted -- provided to the public domain + Version 1.1 25 November 2004 Mark Adler */ + +/* Version history: + 1.0 24 Nov 2004 First version + 1.1 25 Nov 2004 Change deflateInit2() to deflateInit() + Use fixed-size, stack-allocated raw buffers + Simplify code moving compression to subroutines + Use assert() for internal errors + Add detailed description of approach + */ + +/* Approach to just fitting a requested compressed size: + + fitblk performs three compression passes on a portion of the input + data in order to determine how much of that input will compress to + nearly the requested output block size. The first pass generates + enough deflate blocks to produce output to fill the requested + output size plus a specified excess amount (see the EXCESS define + below). The last deflate block may go quite a bit past that, but + is discarded. The second pass decompresses and recompresses just + the compressed data that fit in the requested plus excess sized + buffer. The deflate process is terminated after that amount of + input, which is less than the amount consumed on the first pass. + The last deflate block of the result will be of a comparable size + to the final product, so that the header for that deflate block and + the compression ratio for that block will be about the same as in + the final product. The third compression pass decompresses the + result of the second step, but only the compressed data up to the + requested size minus an amount to allow the compressed stream to + complete (see the MARGIN define below). That will result in a + final compressed stream whose length is less than or equal to the + requested size. Assuming sufficient input and a requested size + greater than a few hundred bytes, the shortfall will typically be + less than ten bytes. + + If the input is short enough that the first compression completes + before filling the requested output size, then that compressed + stream is return with no recompression. + + EXCESS is chosen to be just greater than the shortfall seen in a + two pass approach similar to the above. That shortfall is due to + the last deflate block compressing more efficiently with a smaller + header on the second pass. EXCESS is set to be large enough so + that there is enough uncompressed data for the second pass to fill + out the requested size, and small enough so that the final deflate + block of the second pass will be close in size to the final deflate + block of the third and final pass. MARGIN is chosen to be just + large enough to assure that the final compression has enough room + to complete in all cases. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include "zstd_zlibwrapper.h" + +#define LOG_FITBLK(...) /*printf(__VA_ARGS__)*/ +#define local static + +/* print nastygram and leave */ +local void quit(char *why) +{ + fprintf(stderr, "fitblk abort: %s\n", why); + exit(1); +} + +#define RAWLEN 4096 /* intermediate uncompressed buffer size */ + +/* compress from file to def until provided buffer is full or end of + input reached; return last deflate() return value, or Z_ERRNO if + there was read error on the file */ +local int partcompress(FILE *in, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_SYNC_FLUSH; + do { + def->avail_in = (uInt)fread(raw, 1, RAWLEN, in); + if (ferror(in)) + return Z_ERRNO; + def->next_in = raw; + if (feof(in)) + flush = Z_FINISH; + LOG_FITBLK("partcompress1 avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); + ret = deflate(def, flush); + LOG_FITBLK("partcompress2 ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); + assert(ret != Z_STREAM_ERROR); + } while (def->avail_out != 0 && flush == Z_SYNC_FLUSH); + return ret; +} + +/* recompress from inf's input to def's output; the input for inf and + the output for def are set in those structures before calling; + return last deflate() return value, or Z_MEM_ERROR if inflate() + was not able to allocate enough memory when it needed to */ +local int recompress(z_streamp inf, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_NO_FLUSH; + LOG_FITBLK("recompress start\n"); + do { + /* decompress */ + inf->avail_out = RAWLEN; + inf->next_out = raw; + LOG_FITBLK("recompress1inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out); + ret = inflate(inf, Z_NO_FLUSH); + LOG_FITBLK("recompress2inflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)inf->avail_in, (int)inf->total_in, (int)inf->avail_out, (int)inf->total_out); + assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR && + ret != Z_NEED_DICT); + if (ret == Z_MEM_ERROR) + return ret; + + /* compress what was decompresed until done or no room */ + def->avail_in = RAWLEN - inf->avail_out; + def->next_in = raw; + if (inf->avail_out != 0) + flush = Z_FINISH; + LOG_FITBLK("recompress1deflate avail_in=%d total_in=%d avail_out=%d total_out=%d\n", (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); + ret = deflate(def, flush); + LOG_FITBLK("recompress2deflate ret=%d avail_in=%d total_in=%d avail_out=%d total_out=%d\n", ret, (int)def->avail_in, (int)def->total_in, (int)def->avail_out, (int)def->total_out); + assert(ret != Z_STREAM_ERROR); + } while (ret != Z_STREAM_END && def->avail_out != 0); + return ret; +} + +#define EXCESS 256 /* empirically determined stream overage */ +#define MARGIN 8 /* amount to back off for completion */ + +/* compress from stdin to fixed-size block on stdout */ +int main(int argc, char **argv) +{ + int ret; /* return code */ + unsigned size; /* requested fixed output block size */ + unsigned have; /* bytes written by deflate() call */ + unsigned char *blk; /* intermediate and final stream */ + unsigned char *tmp; /* close to desired size stream */ + z_stream def, inf; /* zlib deflate and inflate states */ + + /* get requested output size */ + if (argc != 2) + quit("need one argument: size of output block"); + ret = (int)strtol(argv[1], argv + 1, 10); + if (argv[1][0] != 0) + quit("argument must be a number"); + if (ret < 8) /* 8 is minimum zlib stream size */ + quit("need positive size of 8 or greater"); + size = (unsigned)ret; + + printf("zlib version %s\n", ZLIB_VERSION); + if (ZWRAP_isUsingZSTDcompression()) printf("zstd version %s\n", zstdVersion()); + + /* allocate memory for buffers and compression engine */ + blk = (unsigned char*)malloc(size + EXCESS); + def.zalloc = Z_NULL; + def.zfree = Z_NULL; + def.opaque = Z_NULL; + ret = deflateInit(&def, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK || blk == NULL) + quit("out of memory"); + + /* compress from stdin until output full, or no more input */ + def.avail_out = size + EXCESS; + def.next_out = blk; + LOG_FITBLK("partcompress1 total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); + ret = partcompress(stdin, &def); + printf("partcompress total_in=%d total_out=%d\n", (int)def.total_in, (int)def.total_out); + if (ret == Z_ERRNO) + quit("error reading input"); + + /* if it all fit, then size was undersubscribed -- done! */ + if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { + /* write block to stdout */ + have = size + EXCESS - def.avail_out; + /* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + * quit("error writing output"); */ + + /* clean up and print results to stderr */ + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (all input)\n", + size - have, size); + return 0; + } + + /* it didn't all fit -- set up for recompression */ + inf.zalloc = Z_NULL; + inf.zfree = Z_NULL; + inf.opaque = Z_NULL; + inf.avail_in = 0; + inf.next_in = Z_NULL; + ret = inflateInit(&inf); + tmp = (unsigned char*)malloc(size + EXCESS); + if (ret != Z_OK || tmp == NULL) + quit("out of memory"); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do first recompression close to the right amount */ + inf.avail_in = size + EXCESS; + inf.next_in = blk; + def.avail_out = size + EXCESS; + def.next_out = tmp; + LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); + ret = recompress(&inf, &def); + LOG_FITBLK("recompress1 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + + /* set up for next recompression */ + ret = inflateReset(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do second and final recompression (third compression) */ + inf.avail_in = size - MARGIN; /* assure stream will complete */ + inf.next_in = tmp; + def.avail_out = size; + def.next_out = blk; + LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); + ret = recompress(&inf, &def); + LOG_FITBLK("recompress2 inf.total_in=%d def.total_out=%d\n", (int)inf.total_in, (int)def.total_out); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */ + + /* done -- write block to stdout */ + have = size - def.avail_out; + /* if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + * quit("error writing output"); */ + + /* clean up and print results to stderr */ + free(tmp); + ret = inflateEnd(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (%lu input)\n", + size - have, size, def.total_in); + return 0; +} diff --git a/src/zstd/zlibWrapper/examples/fitblk_original.c b/src/zstd/zlibWrapper/examples/fitblk_original.c new file mode 100644 index 000000000..20f351bfa --- /dev/null +++ b/src/zstd/zlibWrapper/examples/fitblk_original.c @@ -0,0 +1,233 @@ +/* fitblk.c: example of fitting compressed output to a specified size + Not copyrighted -- provided to the public domain + Version 1.1 25 November 2004 Mark Adler */ + +/* Version history: + 1.0 24 Nov 2004 First version + 1.1 25 Nov 2004 Change deflateInit2() to deflateInit() + Use fixed-size, stack-allocated raw buffers + Simplify code moving compression to subroutines + Use assert() for internal errors + Add detailed description of approach + */ + +/* Approach to just fitting a requested compressed size: + + fitblk performs three compression passes on a portion of the input + data in order to determine how much of that input will compress to + nearly the requested output block size. The first pass generates + enough deflate blocks to produce output to fill the requested + output size plus a specified excess amount (see the EXCESS define + below). The last deflate block may go quite a bit past that, but + is discarded. The second pass decompresses and recompresses just + the compressed data that fit in the requested plus excess sized + buffer. The deflate process is terminated after that amount of + input, which is less than the amount consumed on the first pass. + The last deflate block of the result will be of a comparable size + to the final product, so that the header for that deflate block and + the compression ratio for that block will be about the same as in + the final product. The third compression pass decompresses the + result of the second step, but only the compressed data up to the + requested size minus an amount to allow the compressed stream to + complete (see the MARGIN define below). That will result in a + final compressed stream whose length is less than or equal to the + requested size. Assuming sufficient input and a requested size + greater than a few hundred bytes, the shortfall will typically be + less than ten bytes. + + If the input is short enough that the first compression completes + before filling the requested output size, then that compressed + stream is return with no recompression. + + EXCESS is chosen to be just greater than the shortfall seen in a + two pass approach similar to the above. That shortfall is due to + the last deflate block compressing more efficiently with a smaller + header on the second pass. EXCESS is set to be large enough so + that there is enough uncompressed data for the second pass to fill + out the requested size, and small enough so that the final deflate + block of the second pass will be close in size to the final deflate + block of the third and final pass. MARGIN is chosen to be just + large enough to assure that the final compression has enough room + to complete in all cases. + */ + +#include <stdio.h> +#include <stdlib.h> +#include <assert.h> +#include "zlib.h" + +#define local static + +/* print nastygram and leave */ +local void quit(char *why) +{ + fprintf(stderr, "fitblk abort: %s\n", why); + exit(1); +} + +#define RAWLEN 4096 /* intermediate uncompressed buffer size */ + +/* compress from file to def until provided buffer is full or end of + input reached; return last deflate() return value, or Z_ERRNO if + there was read error on the file */ +local int partcompress(FILE *in, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_NO_FLUSH; + do { + def->avail_in = fread(raw, 1, RAWLEN, in); + if (ferror(in)) + return Z_ERRNO; + def->next_in = raw; + if (feof(in)) + flush = Z_FINISH; + ret = deflate(def, flush); + assert(ret != Z_STREAM_ERROR); + } while (def->avail_out != 0 && flush == Z_NO_FLUSH); + return ret; +} + +/* recompress from inf's input to def's output; the input for inf and + the output for def are set in those structures before calling; + return last deflate() return value, or Z_MEM_ERROR if inflate() + was not able to allocate enough memory when it needed to */ +local int recompress(z_streamp inf, z_streamp def) +{ + int ret, flush; + unsigned char raw[RAWLEN]; + + flush = Z_NO_FLUSH; + do { + /* decompress */ + inf->avail_out = RAWLEN; + inf->next_out = raw; + ret = inflate(inf, Z_NO_FLUSH); + assert(ret != Z_STREAM_ERROR && ret != Z_DATA_ERROR && + ret != Z_NEED_DICT); + if (ret == Z_MEM_ERROR) + return ret; + + /* compress what was decompresed until done or no room */ + def->avail_in = RAWLEN - inf->avail_out; + def->next_in = raw; + if (inf->avail_out != 0) + flush = Z_FINISH; + ret = deflate(def, flush); + assert(ret != Z_STREAM_ERROR); + } while (ret != Z_STREAM_END && def->avail_out != 0); + return ret; +} + +#define EXCESS 256 /* empirically determined stream overage */ +#define MARGIN 8 /* amount to back off for completion */ + +/* compress from stdin to fixed-size block on stdout */ +int main(int argc, char **argv) +{ + int ret; /* return code */ + unsigned size; /* requested fixed output block size */ + unsigned have; /* bytes written by deflate() call */ + unsigned char *blk; /* intermediate and final stream */ + unsigned char *tmp; /* close to desired size stream */ + z_stream def, inf; /* zlib deflate and inflate states */ + + /* get requested output size */ + if (argc != 2) + quit("need one argument: size of output block"); + ret = strtol(argv[1], argv + 1, 10); + if (argv[1][0] != 0) + quit("argument must be a number"); + if (ret < 8) /* 8 is minimum zlib stream size */ + quit("need positive size of 8 or greater"); + size = (unsigned)ret; + + /* allocate memory for buffers and compression engine */ + blk = malloc(size + EXCESS); + def.zalloc = Z_NULL; + def.zfree = Z_NULL; + def.opaque = Z_NULL; + ret = deflateInit(&def, Z_DEFAULT_COMPRESSION); + if (ret != Z_OK || blk == NULL) + quit("out of memory"); + + /* compress from stdin until output full, or no more input */ + def.avail_out = size + EXCESS; + def.next_out = blk; + ret = partcompress(stdin, &def); + if (ret == Z_ERRNO) + quit("error reading input"); + + /* if it all fit, then size was undersubscribed -- done! */ + if (ret == Z_STREAM_END && def.avail_out >= EXCESS) { + /* write block to stdout */ + have = size + EXCESS - def.avail_out; + if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + quit("error writing output"); + + /* clean up and print results to stderr */ + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (all input)\n", + size - have, size); + return 0; + } + + /* it didn't all fit -- set up for recompression */ + inf.zalloc = Z_NULL; + inf.zfree = Z_NULL; + inf.opaque = Z_NULL; + inf.avail_in = 0; + inf.next_in = Z_NULL; + ret = inflateInit(&inf); + tmp = malloc(size + EXCESS); + if (ret != Z_OK || tmp == NULL) + quit("out of memory"); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do first recompression close to the right amount */ + inf.avail_in = size + EXCESS; + inf.next_in = blk; + def.avail_out = size + EXCESS; + def.next_out = tmp; + ret = recompress(&inf, &def); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + + /* set up for next recompression */ + ret = inflateReset(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateReset(&def); + assert(ret != Z_STREAM_ERROR); + + /* do second and final recompression (third compression) */ + inf.avail_in = size - MARGIN; /* assure stream will complete */ + inf.next_in = tmp; + def.avail_out = size; + def.next_out = blk; + ret = recompress(&inf, &def); + if (ret == Z_MEM_ERROR) + quit("out of memory"); + assert(ret == Z_STREAM_END); /* otherwise MARGIN too small */ + + /* done -- write block to stdout */ + have = size - def.avail_out; + if (fwrite(blk, 1, have, stdout) != have || ferror(stdout)) + quit("error writing output"); + + /* clean up and print results to stderr */ + free(tmp); + ret = inflateEnd(&inf); + assert(ret != Z_STREAM_ERROR); + ret = deflateEnd(&def); + assert(ret != Z_STREAM_ERROR); + free(blk); + fprintf(stderr, + "%u bytes unused out of %u requested (%lu input)\n", + size - have, size, def.total_in); + return 0; +} diff --git a/src/zstd/zlibWrapper/examples/minigzip.c b/src/zstd/zlibWrapper/examples/minigzip.c new file mode 100644 index 000000000..f67be0956 --- /dev/null +++ b/src/zstd/zlibWrapper/examples/minigzip.c @@ -0,0 +1,640 @@ +/* minigzip.c contains minimal changes required to be compiled with zlibWrapper: + * - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h" */ + +/* minigzip.c -- simulate gzip using the zlib compression library + * Copyright (C) 1995-2006, 2010, 2011 Jean-loup Gailly. + * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html + */ + +/* + * minigzip is a minimal implementation of the gzip utility. This is + * only an example of using zlib and isn't meant to replace the + * full-featured gzip. No attempt is made to deal with file systems + * limiting names to 14 or 8+3 characters, etc... Error checking is + * very limited. So use minigzip only for testing; use gzip for the + * real thing. On MSDOS, use only on file names without extension + * or in pipe mode. + */ + +/* @(#) $Id$ */ + +#define _POSIX_SOURCE /* fileno */ + +#include "zstd_zlibwrapper.h" +#include <stdio.h> + +#ifdef STDC +# include <string.h> +# include <stdlib.h> +#endif + +#ifdef USE_MMAP +# include <sys/types.h> +# include <sys/mman.h> +# include <sys/stat.h> +#endif + +#if defined(MSDOS) || defined(OS2) || defined(WIN32) || defined(__CYGWIN__) +# include <fcntl.h> +# include <io.h> +# ifdef UNDER_CE +# include <stdlib.h> +# endif +# define SET_BINARY_MODE(file) setmode(fileno(file), O_BINARY) +#else +# define SET_BINARY_MODE(file) +#endif + +#ifdef _MSC_VER +# define snprintf _snprintf +#endif + +#ifdef VMS +# define unlink delete +# define GZ_SUFFIX "-gz" +#endif +#ifdef RISCOS +# define unlink remove +# define GZ_SUFFIX "-gz" +# define fileno(file) file->__file +#endif +#if defined(__MWERKS__) && __dest_os != __be_os && __dest_os != __win32_os +# include <unix.h> /* for fileno */ +#endif + +#if !defined(Z_HAVE_UNISTD_H) && !defined(_LARGEFILE64_SOURCE) +#ifndef WIN32 /* unlink already in stdio.h for WIN32 */ + extern int unlink OF((const char *)); +#endif +#endif + +#if defined(UNDER_CE) +# include <windows.h> +# define perror(s) pwinerror(s) + +/* Map the Windows error number in ERROR to a locale-dependent error + message string and return a pointer to it. Typically, the values + for ERROR come from GetLastError. + + The string pointed to shall not be modified by the application, + but may be overwritten by a subsequent call to strwinerror + + The strwinerror function does not change the current setting + of GetLastError. */ + +static char *strwinerror (error) + DWORD error; +{ + static char buf[1024]; + + wchar_t *msgbuf; + DWORD lasterr = GetLastError(); + DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + error, + 0, /* Default language */ + (LPVOID)&msgbuf, + 0, + NULL); + if (chars != 0) { + /* If there is an \r\n appended, zap it. */ + if (chars >= 2 + && msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') { + chars -= 2; + msgbuf[chars] = 0; + } + + if (chars > sizeof (buf) - 1) { + chars = sizeof (buf) - 1; + msgbuf[chars] = 0; + } + + wcstombs(buf, msgbuf, chars + 1); + LocalFree(msgbuf); + } + else { + sprintf(buf, "unknown win32 error (%ld)", error); + } + + SetLastError(lasterr); + return buf; +} + +static void pwinerror (s) + const char *s; +{ + if (s && *s) + fprintf(stderr, "%s: %s\n", s, strwinerror(GetLastError ())); + else + fprintf(stderr, "%s\n", strwinerror(GetLastError ())); +} + +#endif /* UNDER_CE */ + +#ifndef GZ_SUFFIX +# define GZ_SUFFIX ".gz" +#endif +#define SUFFIX_LEN (sizeof(GZ_SUFFIX)-1) + +#define BUFLEN 16384 +#define MAX_NAME_LEN 1024 + +#ifdef MAXSEG_64K +# define local static + /* Needed for systems with limitation on stack size. */ +#else +# define local +#endif + +#ifdef Z_SOLO +/* for Z_SOLO, create simplified gz* functions using deflate and inflate */ + +#if defined(Z_HAVE_UNISTD_H) || defined(Z_LARGE) +# include <unistd.h> /* for unlink() */ +#endif + +void *myalloc OF((void *, unsigned, unsigned)); +void myfree OF((void *, void *)); + +void *myalloc(q, n, m) + void *q; + unsigned n, m; +{ + q = Z_NULL; + return calloc(n, m); +} + +void myfree(q, p) + void *q, *p; +{ + q = Z_NULL; + free(p); +} + +typedef struct gzFile_s { + FILE *file; + int write; + int err; + char *msg; + z_stream strm; +} *gzFile; + +gzFile gzopen OF((const char *, const char *)); +gzFile gzdopen OF((int, const char *)); +gzFile gz_open OF((const char *, int, const char *)); + +gzFile gzopen(path, mode) +const char *path; +const char *mode; +{ + return gz_open(path, -1, mode); +} + +gzFile gzdopen(fd, mode) +int fd; +const char *mode; +{ + return gz_open(NULL, fd, mode); +} + +gzFile gz_open(path, fd, mode) + const char *path; + int fd; + const char *mode; +{ + gzFile gz; + int ret; + + gz = malloc(sizeof(struct gzFile_s)); + if (gz == NULL) + return NULL; + gz->write = strchr(mode, 'w') != NULL; + gz->strm.zalloc = myalloc; + gz->strm.zfree = myfree; + gz->strm.opaque = Z_NULL; + if (gz->write) + ret = deflateInit2(&(gz->strm), -1, 8, 15 + 16, 8, 0); + else { + gz->strm.next_in = 0; + gz->strm.avail_in = Z_NULL; + ret = inflateInit2(&(gz->strm), 15 + 16); + } + if (ret != Z_OK) { + free(gz); + return NULL; + } + gz->file = path == NULL ? fdopen(fd, gz->write ? "wb" : "rb") : + fopen(path, gz->write ? "wb" : "rb"); + if (gz->file == NULL) { + gz->write ? deflateEnd(&(gz->strm)) : inflateEnd(&(gz->strm)); + free(gz); + return NULL; + } + gz->err = 0; + gz->msg = ""; + return gz; +} + +int gzwrite OF((gzFile, const void *, unsigned)); + +int gzwrite(gz, buf, len) + gzFile gz; + const void *buf; + unsigned len; +{ + z_stream *strm; + unsigned char out[BUFLEN]; + + if (gz == NULL || !gz->write) + return 0; + strm = &(gz->strm); + strm->next_in = (void *)buf; + strm->avail_in = len; + do { + strm->next_out = out; + strm->avail_out = BUFLEN; + (void)deflate(strm, Z_NO_FLUSH); + fwrite(out, 1, BUFLEN - strm->avail_out, gz->file); + } while (strm->avail_out == 0); + return len; +} + +int gzread OF((gzFile, void *, unsigned)); + +int gzread(gz, buf, len) + gzFile gz; + void *buf; + unsigned len; +{ + int ret; + unsigned got; + unsigned char in[1]; + z_stream *strm; + + if (gz == NULL || gz->write) + return 0; + if (gz->err) + return 0; + strm = &(gz->strm); + strm->next_out = (void *)buf; + strm->avail_out = len; + do { + got = fread(in, 1, 1, gz->file); + if (got == 0) + break; + strm->next_in = in; + strm->avail_in = 1; + ret = inflate(strm, Z_NO_FLUSH); + if (ret == Z_DATA_ERROR) { + gz->err = Z_DATA_ERROR; + gz->msg = strm->msg; + return 0; + } + if (ret == Z_STREAM_END) + inflateReset(strm); + } while (strm->avail_out); + return len - strm->avail_out; +} + +int gzclose OF((gzFile)); + +int gzclose(gz) + gzFile gz; +{ + z_stream *strm; + unsigned char out[BUFLEN]; + + if (gz == NULL) + return Z_STREAM_ERROR; + strm = &(gz->strm); + if (gz->write) { + strm->next_in = Z_NULL; + strm->avail_in = 0; + do { + strm->next_out = out; + strm->avail_out = BUFLEN; + (void)deflate(strm, Z_FINISH); + fwrite(out, 1, BUFLEN - strm->avail_out, gz->file); + } while (strm->avail_out == 0); + deflateEnd(strm); + } + else + inflateEnd(strm); + fclose(gz->file); + free(gz); + return Z_OK; +} + +const char *gzerror OF((gzFile, int *)); + +const char *gzerror(gz, err) + gzFile gz; + int *err; +{ + *err = gz->err; + return gz->msg; +} + +#endif + +char *prog; + +void error OF((const char *msg)); +void gz_compress OF((FILE *in, gzFile out)); +#ifdef USE_MMAP +int gz_compress_mmap OF((FILE *in, gzFile out)); +#endif +void gz_uncompress OF((gzFile in, FILE *out)); +void file_compress OF((char *file, char *mode)); +void file_uncompress OF((char *file)); +int main OF((int argc, char *argv[])); + +/* =========================================================================== + * Display error message and exit + */ +void error(msg) + const char *msg; +{ + fprintf(stderr, "%s: %s\n", prog, msg); + exit(1); +} + +/* =========================================================================== + * Compress input to output then close both files. + */ + +void gz_compress(in, out) + FILE *in; + gzFile out; +{ + local char buf[BUFLEN]; + int len; + int err; + +#ifdef USE_MMAP + /* Try first compressing with mmap. If mmap fails (minigzip used in a + * pipe), use the normal fread loop. + */ + if (gz_compress_mmap(in, out) == Z_OK) return; +#endif + for (;;) { + len = (int)fread(buf, 1, sizeof(buf), in); + if (ferror(in)) { + perror("fread"); + exit(1); + } + if (len == 0) break; + + if (gzwrite(out, buf, (unsigned)len) != len) error(gzerror(out, &err)); + } + fclose(in); + if (gzclose(out) != Z_OK) error("failed gzclose"); +} + +#ifdef USE_MMAP /* MMAP version, Miguel Albrecht <malbrech@eso.org> */ + +/* Try compressing the input file at once using mmap. Return Z_OK if + * if success, Z_ERRNO otherwise. + */ +int gz_compress_mmap(in, out) + FILE *in; + gzFile out; +{ + int len; + int err; + int ifd = fileno(in); + caddr_t buf; /* mmap'ed buffer for the entire input file */ + off_t buf_len; /* length of the input file */ + struct stat sb; + + /* Determine the size of the file, needed for mmap: */ + if (fstat(ifd, &sb) < 0) return Z_ERRNO; + buf_len = sb.st_size; + if (buf_len <= 0) return Z_ERRNO; + + /* Now do the actual mmap: */ + buf = mmap((caddr_t) 0, buf_len, PROT_READ, MAP_SHARED, ifd, (off_t)0); + if (buf == (caddr_t)(-1)) return Z_ERRNO; + + /* Compress the whole file at once: */ + len = gzwrite(out, (char *)buf, (unsigned)buf_len); + + if (len != (int)buf_len) error(gzerror(out, &err)); + + munmap(buf, buf_len); + fclose(in); + if (gzclose(out) != Z_OK) error("failed gzclose"); + return Z_OK; +} +#endif /* USE_MMAP */ + +/* =========================================================================== + * Uncompress input to output then close both files. + */ +void gz_uncompress(in, out) + gzFile in; + FILE *out; +{ + local char buf[BUFLEN]; + int len; + int err; + + for (;;) { + len = gzread(in, buf, sizeof(buf)); + if (len < 0) error (gzerror(in, &err)); + if (len == 0) break; + + if ((int)fwrite(buf, 1, (unsigned)len, out) != len) { + error("failed fwrite"); + } + } + if (fclose(out)) error("failed fclose"); + + if (gzclose(in) != Z_OK) error("failed gzclose"); +} + + +/* =========================================================================== + * Compress the given file: create a corresponding .gz file and remove the + * original. + */ +void file_compress(file, mode) + char *file; + char *mode; +{ + local char outfile[MAX_NAME_LEN]; + FILE *in; + gzFile out; + + if (strlen(file) + strlen(GZ_SUFFIX) >= sizeof(outfile)) { + fprintf(stderr, "%s: filename too long\n", prog); + exit(1); + } + + strcpy(outfile, file); + strcat(outfile, GZ_SUFFIX); + + in = fopen(file, "rb"); + if (in == NULL) { + perror(file); + exit(1); + } + out = gzopen(outfile, mode); + if (out == NULL) { + fprintf(stderr, "%s: can't gzopen %s\n", prog, outfile); + exit(1); + } + gz_compress(in, out); + + unlink(file); +} + + +/* =========================================================================== + * Uncompress the given file and remove the original. + */ +void file_uncompress(file) + char *file; +{ + local char buf[MAX_NAME_LEN]; + char *infile, *outfile; + FILE *out; + gzFile in; + size_t len = strlen(file); + + if (len + strlen(GZ_SUFFIX) >= sizeof(buf)) { + fprintf(stderr, "%s: filename too long\n", prog); + exit(1); + } + + strcpy(buf, file); + + if (len > SUFFIX_LEN && strcmp(file+len-SUFFIX_LEN, GZ_SUFFIX) == 0) { + infile = file; + outfile = buf; + outfile[len-3] = '\0'; + } else { + outfile = file; + infile = buf; + strcat(infile, GZ_SUFFIX); + } + in = gzopen(infile, "rb"); + if (in == NULL) { + fprintf(stderr, "%s: can't gzopen %s\n", prog, infile); + exit(1); + } + out = fopen(outfile, "wb"); + if (out == NULL) { + perror(file); + exit(1); + } + + gz_uncompress(in, out); + + unlink(infile); +} + + +/* =========================================================================== + * Usage: minigzip [-c] [-d] [-f] [-h] [-r] [-1 to -9] [files...] + * -c : write to standard output + * -d : decompress + * -f : compress with Z_FILTERED + * -h : compress with Z_HUFFMAN_ONLY + * -r : compress with Z_RLE + * -1 to -9 : compression level + */ + +int main(argc, argv) + int argc; + char *argv[]; +{ + int copyout = 0; + int uncompr = 0; + gzFile file; + char *bname, outmode[20]; + + strcpy(outmode, "wb6 "); + + prog = argv[0]; + bname = strrchr(argv[0], '/'); + if (bname) + bname++; + else + bname = argv[0]; + argc--, argv++; + + if (!strcmp(bname, "gunzip")) + uncompr = 1; + else if (!strcmp(bname, "zcat")) + copyout = uncompr = 1; + + while (argc > 0) { + if (strcmp(*argv, "-c") == 0) + copyout = 1; + else if (strcmp(*argv, "-d") == 0) + uncompr = 1; + else if (strcmp(*argv, "-f") == 0) + outmode[3] = 'f'; + else if (strcmp(*argv, "-h") == 0) + outmode[3] = 'h'; + else if (strcmp(*argv, "-r") == 0) + outmode[3] = 'R'; + else if ((*argv)[0] == '-' && (*argv)[1] >= '1' && (*argv)[1] <= '9' && + (*argv)[2] == 0) + outmode[2] = (*argv)[1]; + else + break; + argc--, argv++; + } + if (outmode[3] == ' ') + outmode[3] = 0; + if (argc == 0) { + SET_BINARY_MODE(stdin); + SET_BINARY_MODE(stdout); + if (uncompr) { + file = gzdopen(fileno(stdin), "rb"); + if (file == NULL) error("can't gzdopen stdin"); + gz_uncompress(file, stdout); + } else { + file = gzdopen(fileno(stdout), outmode); + if (file == NULL) error("can't gzdopen stdout"); + gz_compress(stdin, file); + } + } else { + if (copyout) { + SET_BINARY_MODE(stdout); + } + do { + if (uncompr) { + if (copyout) { + file = gzopen(*argv, "rb"); + if (file == NULL) + fprintf(stderr, "%s: can't gzopen %s\n", prog, *argv); + else + gz_uncompress(file, stdout); + } else { + file_uncompress(*argv); + } + } else { + if (copyout) { + FILE * in = fopen(*argv, "rb"); + + if (in == NULL) { + perror(*argv); + } else { + file = gzdopen(fileno(stdout), outmode); + if (file == NULL) error("can't gzdopen stdout"); + + gz_compress(in, file); + } + + } else { + file_compress(*argv, outmode); + } + } + } while (argv++, --argc); + } + return 0; +} diff --git a/src/zstd/zlibWrapper/examples/zwrapbench.c b/src/zstd/zlibWrapper/examples/zwrapbench.c new file mode 100644 index 000000000..f30cad40c --- /dev/null +++ b/src/zstd/zlibWrapper/examples/zwrapbench.c @@ -0,0 +1,1001 @@ +/* + * Copyright (c) 2016-present, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + */ + + +/* ************************************* +* Includes +***************************************/ +#include "util.h" /* Compiler options, UTIL_GetFileSize, UTIL_sleep */ +#include <stdlib.h> /* malloc, free */ +#include <string.h> /* memset */ +#include <stdio.h> /* fprintf, fopen, ftello64 */ +#include <time.h> /* clock_t, clock, CLOCKS_PER_SEC */ +#include <ctype.h> /* toupper */ +#include <errno.h> /* errno */ + +#include "timefn.h" /* UTIL_time_t, UTIL_getTime, UTIL_clockSpanMicro, UTIL_waitForNextTick */ +#include "mem.h" +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" +#include "datagen.h" /* RDG_genBuffer */ +#include "xxhash.h" + +#include "zstd_zlibwrapper.h" + + + +/*-************************************ +* Tuning parameters +**************************************/ +#ifndef ZSTDCLI_CLEVEL_DEFAULT +# define ZSTDCLI_CLEVEL_DEFAULT 3 +#endif + + +/*-************************************ +* Constants +**************************************/ +#define COMPRESSOR_NAME "Zstandard wrapper for zlib command line interface" +#ifndef ZSTD_VERSION +# define ZSTD_VERSION "v" ZSTD_VERSION_STRING +#endif +#define AUTHOR "Yann Collet" +#define WELCOME_MESSAGE "*** %s %i-bits %s, by %s ***\n", COMPRESSOR_NAME, (int)(sizeof(size_t)*8), ZSTD_VERSION, AUTHOR + +#ifndef ZSTD_GIT_COMMIT +# define ZSTD_GIT_COMMIT_STRING "" +#else +# define ZSTD_GIT_COMMIT_STRING ZSTD_EXPAND_AND_QUOTE(ZSTD_GIT_COMMIT) +#endif + +#define NBLOOPS 3 +#define TIMELOOP_MICROSEC 1*1000000ULL /* 1 second */ +#define ACTIVEPERIOD_MICROSEC 70*1000000ULL /* 70 seconds */ +#define COOLPERIOD_SEC 10 + +#define KB *(1 <<10) +#define MB *(1 <<20) +#define GB *(1U<<30) + +static const size_t maxMemory = (sizeof(size_t)==4) ? (2 GB - 64 MB) : (size_t)(1ULL << ((sizeof(size_t)*8)-31)); + +static U32 g_compressibilityDefault = 50; + + +/* ************************************* +* console display +***************************************/ +#define DEFAULT_DISPLAY_LEVEL 2 +#define DISPLAY(...) fprintf(displayOut, __VA_ARGS__) +#define DISPLAYLEVEL(l, ...) if (g_displayLevel>=l) { DISPLAY(__VA_ARGS__); } +static unsigned g_displayLevel = DEFAULT_DISPLAY_LEVEL; /* 0 : no display; 1: errors; 2 : + result + interaction + warnings; 3 : + progression; 4 : + information */ +static FILE* displayOut; + +#define DISPLAYUPDATE(l, ...) if (g_displayLevel>=l) { \ + if ((clock() - g_time > refreshRate) || (g_displayLevel>=4)) \ + { g_time = clock(); DISPLAY(__VA_ARGS__); \ + if (g_displayLevel>=4) fflush(displayOut); } } +static const clock_t refreshRate = CLOCKS_PER_SEC * 15 / 100; +static clock_t g_time = 0; + + +/* ************************************* +* Exceptions +***************************************/ +#ifndef DEBUG +# define DEBUG 0 +#endif +#define DEBUGOUTPUT(...) { if (DEBUG) DISPLAY(__VA_ARGS__); } +#define EXM_THROW(error, ...) \ +{ \ + DEBUGOUTPUT("Error defined at %s, line %i : \n", __FILE__, __LINE__); \ + DISPLAYLEVEL(1, "Error %i : ", error); \ + DISPLAYLEVEL(1, __VA_ARGS__); \ + DISPLAYLEVEL(1, "\n"); \ + exit(error); \ +} + + +/* ************************************* +* Benchmark Parameters +***************************************/ +static unsigned g_nbIterations = NBLOOPS; +static size_t g_blockSize = 0; +int g_additionalParam = 0; + +void BMK_setNotificationLevel(unsigned level) { g_displayLevel=level; } + +void BMK_setAdditionalParam(int additionalParam) { g_additionalParam=additionalParam; } + +void BMK_SetNbIterations(unsigned nbLoops) +{ + g_nbIterations = nbLoops; + DISPLAYLEVEL(3, "- test >= %u seconds per compression / decompression -\n", g_nbIterations); +} + +void BMK_SetBlockSize(size_t blockSize) +{ + g_blockSize = blockSize; + DISPLAYLEVEL(2, "using blocks of size %u KB \n", (unsigned)(blockSize>>10)); +} + + +/* ******************************************************** +* Bench functions +**********************************************************/ +#undef MIN +#undef MAX +#define MIN(a,b) ((a)<(b) ? (a) : (b)) +#define MAX(a,b) ((a)>(b) ? (a) : (b)) + +typedef struct +{ + z_const char* srcPtr; + size_t srcSize; + char* cPtr; + size_t cRoom; + size_t cSize; + char* resPtr; + size_t resSize; +} blockParam_t; + +typedef enum { BMK_ZSTD, BMK_ZSTD_STREAM, BMK_ZLIB, BMK_ZWRAP_ZLIB, BMK_ZWRAP_ZSTD, BMK_ZLIB_REUSE, BMK_ZWRAP_ZLIB_REUSE, BMK_ZWRAP_ZSTD_REUSE } BMK_compressor; + + +static int BMK_benchMem(z_const void* srcBuffer, size_t srcSize, + const char* displayName, int cLevel, + const size_t* fileSizes, U32 nbFiles, + const void* dictBuffer, size_t dictBufferSize, BMK_compressor compressor) +{ + size_t const blockSize = (g_blockSize>=32 ? g_blockSize : srcSize) + (!srcSize) /* avoid div by 0 */ ; + size_t const avgSize = MIN(g_blockSize, (srcSize / nbFiles)); + U32 const maxNbBlocks = (U32) ((srcSize + (blockSize-1)) / blockSize) + nbFiles; + blockParam_t* const blockTable = (blockParam_t*) malloc(maxNbBlocks * sizeof(blockParam_t)); + size_t const maxCompressedSize = ZSTD_compressBound(srcSize) + (maxNbBlocks * 1024); /* add some room for safety */ + void* const compressedBuffer = malloc(maxCompressedSize); + void* const resultBuffer = malloc(srcSize); + ZSTD_CCtx* const ctx = ZSTD_createCCtx(); + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + U32 nbBlocks; + + /* checks */ + if (!compressedBuffer || !resultBuffer || !blockTable || !ctx || !dctx) + EXM_THROW(31, "allocation error : not enough memory"); + + /* init */ + if (strlen(displayName)>17) displayName += strlen(displayName)-17; /* can only display 17 characters */ + + /* Init blockTable data */ + { z_const char* srcPtr = (z_const char*)srcBuffer; + char* cPtr = (char*)compressedBuffer; + char* resPtr = (char*)resultBuffer; + U32 fileNb; + for (nbBlocks=0, fileNb=0; fileNb<nbFiles; fileNb++) { + size_t remaining = fileSizes[fileNb]; + U32 const nbBlocksforThisFile = (U32)((remaining + (blockSize-1)) / blockSize); + U32 const blockEnd = nbBlocks + nbBlocksforThisFile; + for ( ; nbBlocks<blockEnd; nbBlocks++) { + size_t const thisBlockSize = MIN(remaining, blockSize); + blockTable[nbBlocks].srcPtr = srcPtr; + blockTable[nbBlocks].cPtr = cPtr; + blockTable[nbBlocks].resPtr = resPtr; + blockTable[nbBlocks].srcSize = thisBlockSize; + blockTable[nbBlocks].cRoom = ZSTD_compressBound(thisBlockSize); + srcPtr += thisBlockSize; + cPtr += blockTable[nbBlocks].cRoom; + resPtr += thisBlockSize; + remaining -= thisBlockSize; + } } } + + /* warming up memory */ + RDG_genBuffer(compressedBuffer, maxCompressedSize, 0.10, 0.50, 1); + + /* Bench */ + { U64 fastestC = (U64)(-1LL), fastestD = (U64)(-1LL); + U64 const crcOrig = XXH64(srcBuffer, srcSize, 0); + UTIL_time_t coolTime; + U64 const maxTime = (g_nbIterations * TIMELOOP_MICROSEC) + 100; + U64 totalCTime=0, totalDTime=0; + U32 cCompleted=0, dCompleted=0; +# define NB_MARKS 4 + const char* const marks[NB_MARKS] = { " |", " /", " =", "\\" }; + U32 markNb = 0; + size_t cSize = 0; + double ratio = 0.; + + coolTime = UTIL_getTime(); + DISPLAYLEVEL(2, "\r%79s\r", ""); + while (!cCompleted | !dCompleted) { + UTIL_time_t clockStart; + U64 clockLoop = g_nbIterations ? TIMELOOP_MICROSEC : 1; + + /* overheat protection */ + if (UTIL_clockSpanMicro(coolTime) > ACTIVEPERIOD_MICROSEC) { + DISPLAYLEVEL(2, "\rcooling down ... \r"); + UTIL_sleep(COOLPERIOD_SEC); + coolTime = UTIL_getTime(); + } + + /* Compression */ + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->\r", marks[markNb], displayName, (unsigned)srcSize); + if (!cCompleted) memset(compressedBuffer, 0xE5, maxCompressedSize); /* warm up and erase result buffer */ + + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); + + if (!cCompleted) { /* still some time to do compression tests */ + U32 nbLoops = 0; + if (compressor == BMK_ZSTD) { + ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); + ZSTD_customMem const cmem = { NULL, NULL, NULL }; + ZSTD_CDict* const cdict = ZSTD_createCDict_advanced(dictBuffer, dictBufferSize, ZSTD_dlm_byRef, ZSTD_dct_auto, zparams.cParams, cmem); + if (cdict==NULL) EXM_THROW(1, "ZSTD_createCDict_advanced() allocation failure"); + + do { + U32 blockNb; + size_t rSize; + for (blockNb=0; blockNb<nbBlocks; blockNb++) { + if (dictBufferSize) { + rSize = ZSTD_compress_usingCDict(ctx, + blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, + blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, + cdict); + } else { + rSize = ZSTD_compressCCtx (ctx, + blockTable[blockNb].cPtr, blockTable[blockNb].cRoom, + blockTable[blockNb].srcPtr,blockTable[blockNb].srcSize, cLevel); + } + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compress_usingCDict() failed : %s", ZSTD_getErrorName(rSize)); + blockTable[blockNb].cSize = rSize; + } + nbLoops++; + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + ZSTD_freeCDict(cdict); + } else if (compressor == BMK_ZSTD_STREAM) { + ZSTD_parameters const zparams = ZSTD_getParams(cLevel, avgSize, dictBufferSize); + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; + ZSTD_CStream* zbc = ZSTD_createCStream(); + size_t rSize; + if (zbc == NULL) EXM_THROW(1, "ZSTD_createCStream() allocation failure"); + rSize = ZSTD_initCStream_advanced(zbc, dictBuffer, dictBufferSize, zparams, avgSize); + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initCStream_advanced() failed : %s", ZSTD_getErrorName(rSize)); + do { + U32 blockNb; + for (blockNb=0; blockNb<nbBlocks; blockNb++) { + rSize = ZSTD_resetCStream(zbc, blockTable[blockNb].srcSize); + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetCStream() failed : %s", ZSTD_getErrorName(rSize)); + inBuffer.src = blockTable[blockNb].srcPtr; + inBuffer.size = blockTable[blockNb].srcSize; + inBuffer.pos = 0; + outBuffer.dst = blockTable[blockNb].cPtr; + outBuffer.size = blockTable[blockNb].cRoom; + outBuffer.pos = 0; + rSize = ZSTD_compressStream(zbc, &outBuffer, &inBuffer); + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_compressStream() failed : %s", ZSTD_getErrorName(rSize)); + rSize = ZSTD_endStream(zbc, &outBuffer); + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_endStream() failed : %s", ZSTD_getErrorName(rSize)); + blockTable[blockNb].cSize = outBuffer.pos; + } + nbLoops++; + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + ZSTD_freeCStream(zbc); + } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) { + z_stream def; + int ret; + int useSetDict = (dictBuffer != NULL); + if (compressor == BMK_ZLIB_REUSE || compressor == BMK_ZWRAP_ZLIB_REUSE) ZWRAP_useZSTDcompression(0); + else ZWRAP_useZSTDcompression(1); + def.zalloc = Z_NULL; + def.zfree = Z_NULL; + def.opaque = Z_NULL; + ret = deflateInit(&def, cLevel); + if (ret != Z_OK) EXM_THROW(1, "deflateInit failure"); + /* if (ZWRAP_isUsingZSTDcompression()) { + ret = ZWRAP_setPledgedSrcSize(&def, avgSize); + if (ret != Z_OK) EXM_THROW(1, "ZWRAP_setPledgedSrcSize failure"); + } */ + do { + U32 blockNb; + for (blockNb=0; blockNb<nbBlocks; blockNb++) { + if (ZWRAP_isUsingZSTDcompression()) + ret = ZWRAP_deflateReset_keepDict(&def); /* reuse dictionary to make compression faster */ + else + ret = deflateReset(&def); + if (ret != Z_OK) EXM_THROW(1, "deflateReset failure"); + if (useSetDict) { + ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize); + if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); + if (ZWRAP_isUsingZSTDcompression()) useSetDict = 0; /* zstd doesn't require deflateSetDictionary after ZWRAP_deflateReset_keepDict */ + } + def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr; + def.avail_in = (uInt)blockTable[blockNb].srcSize; + def.total_in = 0; + def.next_out = (z_Bytef*) blockTable[blockNb].cPtr; + def.avail_out = (uInt)blockTable[blockNb].cRoom; + def.total_out = 0; + ret = deflate(&def, Z_FINISH); + if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure ret=%d srcSize=%d" , ret, (int)blockTable[blockNb].srcSize); + blockTable[blockNb].cSize = def.total_out; + } + nbLoops++; + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + ret = deflateEnd(&def); + if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure"); + } else { + z_stream def; + if (compressor == BMK_ZLIB || compressor == BMK_ZWRAP_ZLIB) ZWRAP_useZSTDcompression(0); + else ZWRAP_useZSTDcompression(1); + do { + U32 blockNb; + for (blockNb=0; blockNb<nbBlocks; blockNb++) { + int ret; + def.zalloc = Z_NULL; + def.zfree = Z_NULL; + def.opaque = Z_NULL; + ret = deflateInit(&def, cLevel); + if (ret != Z_OK) EXM_THROW(1, "deflateInit failure"); + if (dictBuffer) { + ret = deflateSetDictionary(&def, (const z_Bytef*)dictBuffer, dictBufferSize); + if (ret != Z_OK) EXM_THROW(1, "deflateSetDictionary failure"); + } + def.next_in = (z_const z_Bytef*) blockTable[blockNb].srcPtr; + def.avail_in = (uInt)blockTable[blockNb].srcSize; + def.total_in = 0; + def.next_out = (z_Bytef*) blockTable[blockNb].cPtr; + def.avail_out = (uInt)blockTable[blockNb].cRoom; + def.total_out = 0; + ret = deflate(&def, Z_FINISH); + if (ret != Z_STREAM_END) EXM_THROW(1, "deflate failure"); + ret = deflateEnd(&def); + if (ret != Z_OK) EXM_THROW(1, "deflateEnd failure"); + blockTable[blockNb].cSize = def.total_out; + } + nbLoops++; + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + } + { U64 const clockSpan = UTIL_clockSpanMicro(clockStart); + if (clockSpan < fastestC*nbLoops) fastestC = clockSpan / nbLoops; + totalCTime += clockSpan; + cCompleted = totalCTime>maxTime; + } } + + cSize = 0; + { U32 blockNb; for (blockNb=0; blockNb<nbBlocks; blockNb++) cSize += blockTable[blockNb].cSize; } + ratio = (double)srcSize / (double)cSize; + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s\r", + marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio, + (double)srcSize / fastestC ); + + (void)fastestD; (void)crcOrig; /* unused when decompression disabled */ +#if 1 + /* Decompression */ + if (!dCompleted) memset(resultBuffer, 0xD6, srcSize); /* warm result buffer */ + + UTIL_sleepMilli(1); /* give processor time to other processes */ + UTIL_waitForNextTick(); + clockStart = UTIL_getTime(); + + if (!dCompleted) { + U32 nbLoops = 0; + if (compressor == BMK_ZSTD) { + ZSTD_DDict* ddict = ZSTD_createDDict(dictBuffer, dictBufferSize); + if (!ddict) EXM_THROW(2, "ZSTD_createDDict() allocation failure"); + do { + unsigned blockNb; + for (blockNb=0; blockNb<nbBlocks; blockNb++) { + size_t const regenSize = ZSTD_decompress_usingDDict(dctx, + blockTable[blockNb].resPtr, blockTable[blockNb].srcSize, + blockTable[blockNb].cPtr, blockTable[blockNb].cSize, + ddict); + if (ZSTD_isError(regenSize)) { + DISPLAY("ZSTD_decompress_usingDDict() failed on block %u : %s \n", + blockNb, ZSTD_getErrorName(regenSize)); + clockLoop = 0; /* force immediate test end */ + break; + } + blockTable[blockNb].resSize = regenSize; + } + nbLoops++; + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + ZSTD_freeDDict(ddict); + } else if (compressor == BMK_ZSTD_STREAM) { + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; + ZSTD_DStream* zbd = ZSTD_createDStream(); + size_t rSize; + if (zbd == NULL) EXM_THROW(1, "ZSTD_createDStream() allocation failure"); + rSize = ZSTD_initDStream_usingDict(zbd, dictBuffer, dictBufferSize); + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_initDStream() failed : %s", ZSTD_getErrorName(rSize)); + do { + U32 blockNb; + for (blockNb=0; blockNb<nbBlocks; blockNb++) { + rSize = ZSTD_resetDStream(zbd); + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_resetDStream() failed : %s", ZSTD_getErrorName(rSize)); + inBuffer.src = blockTable[blockNb].cPtr; + inBuffer.size = blockTable[blockNb].cSize; + inBuffer.pos = 0; + outBuffer.dst = blockTable[blockNb].resPtr; + outBuffer.size = blockTable[blockNb].srcSize; + outBuffer.pos = 0; + rSize = ZSTD_decompressStream(zbd, &outBuffer, &inBuffer); + if (ZSTD_isError(rSize)) EXM_THROW(1, "ZSTD_decompressStream() failed : %s", ZSTD_getErrorName(rSize)); + blockTable[blockNb].resSize = outBuffer.pos; + } + nbLoops++; + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + ZSTD_freeDStream(zbd); + } else if (compressor == BMK_ZWRAP_ZLIB_REUSE || compressor == BMK_ZWRAP_ZSTD_REUSE || compressor == BMK_ZLIB_REUSE) { + z_stream inf; + int ret; + if (compressor == BMK_ZLIB_REUSE) ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB); + else ZWRAP_setDecompressionType(ZWRAP_AUTO); + inf.zalloc = Z_NULL; + inf.zfree = Z_NULL; + inf.opaque = Z_NULL; + ret = inflateInit(&inf); + if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); + do { + U32 blockNb; + for (blockNb=0; blockNb<nbBlocks; blockNb++) { + if (ZWRAP_isUsingZSTDdecompression(&inf)) + ret = ZWRAP_inflateReset_keepDict(&inf); /* reuse dictionary to make decompression faster; inflate will return Z_NEED_DICT only for the first time */ + else + ret = inflateReset(&inf); + if (ret != Z_OK) EXM_THROW(1, "inflateReset failure"); + inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr; + inf.avail_in = (uInt)blockTable[blockNb].cSize; + inf.total_in = 0; + inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr; + inf.avail_out = (uInt)blockTable[blockNb].srcSize; + inf.total_out = 0; + ret = inflate(&inf, Z_FINISH); + if (ret == Z_NEED_DICT) { + ret = inflateSetDictionary(&inf, (const z_Bytef*)dictBuffer, dictBufferSize); + if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure"); + ret = inflate(&inf, Z_FINISH); + } + if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure"); + blockTable[blockNb].resSize = inf.total_out; + } + nbLoops++; + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + ret = inflateEnd(&inf); + if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure"); + } else { + z_stream inf; + if (compressor == BMK_ZLIB) ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB); + else ZWRAP_setDecompressionType(ZWRAP_AUTO); + do { + U32 blockNb; + for (blockNb=0; blockNb<nbBlocks; blockNb++) { + int ret; + inf.zalloc = Z_NULL; + inf.zfree = Z_NULL; + inf.opaque = Z_NULL; + ret = inflateInit(&inf); + if (ret != Z_OK) EXM_THROW(1, "inflateInit failure"); + inf.next_in = (z_const z_Bytef*) blockTable[blockNb].cPtr; + inf.avail_in = (uInt)blockTable[blockNb].cSize; + inf.total_in = 0; + inf.next_out = (z_Bytef*) blockTable[blockNb].resPtr; + inf.avail_out = (uInt)blockTable[blockNb].srcSize; + inf.total_out = 0; + ret = inflate(&inf, Z_FINISH); + if (ret == Z_NEED_DICT) { + ret = inflateSetDictionary(&inf, (const z_Bytef*) dictBuffer, dictBufferSize); + if (ret != Z_OK) EXM_THROW(1, "inflateSetDictionary failure"); + ret = inflate(&inf, Z_FINISH); + } + if (ret != Z_STREAM_END) EXM_THROW(1, "inflate failure"); + ret = inflateEnd(&inf); + if (ret != Z_OK) EXM_THROW(1, "inflateEnd failure"); + blockTable[blockNb].resSize = inf.total_out; + } + nbLoops++; + } while (UTIL_clockSpanMicro(clockStart) < clockLoop); + } + { U64 const clockSpan = UTIL_clockSpanMicro(clockStart); + if (clockSpan < fastestD*nbLoops) fastestD = clockSpan / nbLoops; + totalDTime += clockSpan; + dCompleted = totalDTime>maxTime; + } } + + markNb = (markNb+1) % NB_MARKS; + DISPLAYLEVEL(2, "%2s-%-17.17s :%10u ->%10u (%5.3f),%6.1f MB/s ,%6.1f MB/s\r", + marks[markNb], displayName, (unsigned)srcSize, (unsigned)cSize, ratio, + (double)srcSize / fastestC, + (double)srcSize / fastestD ); + + /* CRC Checking */ + { U64 const crcCheck = XXH64(resultBuffer, srcSize, 0); + if (crcOrig!=crcCheck) { + size_t u; + DISPLAY("!!! WARNING !!! %14s : Invalid Checksum : %x != %x \n", displayName, (unsigned)crcOrig, (unsigned)crcCheck); + for (u=0; u<srcSize; u++) { + if (((const BYTE*)srcBuffer)[u] != ((const BYTE*)resultBuffer)[u]) { + unsigned segNb, bNb, pos; + size_t bacc = 0; + DISPLAY("Decoding error at pos %u ", (unsigned)u); + for (segNb = 0; segNb < nbBlocks; segNb++) { + if (bacc + blockTable[segNb].srcSize > u) break; + bacc += blockTable[segNb].srcSize; + } + pos = (U32)(u - bacc); + bNb = pos / (128 KB); + DISPLAY("(block %u, sub %u, pos %u) \n", segNb, bNb, pos); + break; + } + if (u==srcSize-1) { /* should never happen */ + DISPLAY("no difference detected\n"); + } } + break; + } } /* CRC Checking */ +#endif + } /* for (testNb = 1; testNb <= (g_nbIterations + !g_nbIterations); testNb++) */ + + if (g_displayLevel == 1) { + double cSpeed = (double)srcSize / fastestC; + double dSpeed = (double)srcSize / fastestD; + if (g_additionalParam) + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s (param=%d)\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName, g_additionalParam); + else + DISPLAY("-%-3i%11i (%5.3f) %6.2f MB/s %6.1f MB/s %s\n", cLevel, (int)cSize, ratio, cSpeed, dSpeed, displayName); + } + DISPLAYLEVEL(2, "%2i#\n", cLevel); + } /* Bench */ + + /* clean up */ + free(blockTable); + free(compressedBuffer); + free(resultBuffer); + ZSTD_freeCCtx(ctx); + ZSTD_freeDCtx(dctx); + return 0; +} + + +static size_t BMK_findMaxMem(U64 requiredMem) +{ + size_t const step = 64 MB; + BYTE* testmem = NULL; + + requiredMem = (((requiredMem >> 26) + 1) << 26); + requiredMem += step; + if (requiredMem > maxMemory) requiredMem = maxMemory; + + do { + testmem = (BYTE*)malloc((size_t)requiredMem); + requiredMem -= step; + } while (!testmem && requiredMem); /* do not allocate zero bytes */ + + free(testmem); + return (size_t)(requiredMem+1); /* avoid zero */ +} + +static void BMK_benchCLevel(void* srcBuffer, size_t benchedSize, + const char* displayName, int cLevel, int cLevelLast, + const size_t* fileSizes, unsigned nbFiles, + const void* dictBuffer, size_t dictBufferSize) +{ + int l; + + const char* pch = strrchr(displayName, '\\'); /* Windows */ + if (!pch) pch = strrchr(displayName, '/'); /* Linux */ + if (pch) displayName = pch+1; + + SET_REALTIME_PRIORITY; + + if (g_displayLevel == 1 && !g_additionalParam) + DISPLAY("bench %s %s: input %u bytes, %u seconds, %u KB blocks\n", + ZSTD_VERSION_STRING, ZSTD_GIT_COMMIT_STRING, + (unsigned)benchedSize, g_nbIterations, (unsigned)(g_blockSize>>10)); + + if (cLevelLast < cLevel) cLevelLast = cLevel; + + DISPLAY("benchmarking zstd %s (using ZSTD_CStream)\n", ZSTD_VERSION_STRING); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZSTD_STREAM); + } + + DISPLAY("benchmarking zstd %s (using ZSTD_CCtx)\n", ZSTD_VERSION_STRING); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZSTD); + } + + DISPLAY("benchmarking zstd %s (using zlibWrapper)\n", ZSTD_VERSION_STRING); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD_REUSE); + } + + DISPLAY("benchmarking zstd %s (zlibWrapper not reusing a context)\n", ZSTD_VERSION_STRING); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZSTD); + } + + + if (cLevelLast > Z_BEST_COMPRESSION) cLevelLast = Z_BEST_COMPRESSION; + + DISPLAY("\n"); + DISPLAY("benchmarking zlib %s\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZLIB_REUSE); + } + + DISPLAY("benchmarking zlib %s (zlib not reusing a context)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZLIB); + } + + DISPLAY("benchmarking zlib %s (using zlibWrapper)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB_REUSE); + } + + DISPLAY("benchmarking zlib %s (zlibWrapper not reusing a context)\n", ZLIB_VERSION); + for (l=cLevel; l <= cLevelLast; l++) { + BMK_benchMem(srcBuffer, benchedSize, + displayName, l, + fileSizes, nbFiles, + dictBuffer, dictBufferSize, BMK_ZWRAP_ZLIB); + } +} + + +/*! BMK_loadFiles() : + Loads `buffer` with content of files listed within `fileNamesTable`. + At most, fills `buffer` entirely */ +static void BMK_loadFiles(void* buffer, size_t bufferSize, + size_t* fileSizes, + const char** fileNamesTable, unsigned nbFiles) +{ + size_t pos = 0, totalSize = 0; + unsigned n; + for (n=0; n<nbFiles; n++) { + FILE* f; + U64 fileSize = UTIL_getFileSize(fileNamesTable[n]); + if (UTIL_isDirectory(fileNamesTable[n])) { + DISPLAYLEVEL(2, "Ignoring %s directory... \n", fileNamesTable[n]); + fileSizes[n] = 0; + continue; + } + if (fileSize == UTIL_FILESIZE_UNKNOWN) { + DISPLAYLEVEL(2, "Cannot determine size of %s ... \n", fileNamesTable[n]); + fileSizes[n] = 0; + continue; + } + f = fopen(fileNamesTable[n], "rb"); + if (f==NULL) EXM_THROW(10, "impossible to open file %s", fileNamesTable[n]); + DISPLAYUPDATE(2, "Loading %s... \r", fileNamesTable[n]); + if (fileSize > bufferSize-pos) fileSize = bufferSize-pos, nbFiles=n; /* buffer too small - stop after this file */ + { size_t const readSize = fread(((char*)buffer)+pos, 1, (size_t)fileSize, f); + if (readSize != (size_t)fileSize) EXM_THROW(11, "could not read %s", fileNamesTable[n]); + pos += readSize; } + fileSizes[n] = (size_t)fileSize; + totalSize += (size_t)fileSize; + fclose(f); + } + + if (totalSize == 0) EXM_THROW(12, "no data to bench"); +} + +static void BMK_benchFileTable(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, int cLevel, int cLevelLast) +{ + void* srcBuffer; + size_t benchedSize; + void* dictBuffer = NULL; + size_t dictBufferSize = 0; + size_t* fileSizes = (size_t*)malloc(nbFiles * sizeof(size_t)); + U64 const totalSizeToLoad = UTIL_getTotalFileSize(fileNamesTable, nbFiles); + char mfName[20] = {0}; + + if (!fileSizes) EXM_THROW(12, "not enough memory for fileSizes"); + + /* Load dictionary */ + if (dictFileName != NULL) { + U64 const dictFileSize = UTIL_getFileSize(dictFileName); + if (dictFileSize > 64 MB) + EXM_THROW(10, "dictionary file %s too large", dictFileName); + dictBufferSize = (size_t)dictFileSize; + dictBuffer = malloc(dictBufferSize); + if (dictBuffer==NULL) + EXM_THROW(11, "not enough memory for dictionary (%u bytes)", (unsigned)dictBufferSize); + BMK_loadFiles(dictBuffer, dictBufferSize, fileSizes, &dictFileName, 1); + } + + /* Memory allocation & restrictions */ + benchedSize = BMK_findMaxMem(totalSizeToLoad * 3) / 3; + if ((U64)benchedSize > totalSizeToLoad) benchedSize = (size_t)totalSizeToLoad; + if (benchedSize < totalSizeToLoad) + DISPLAY("Not enough memory; testing %u MB only...\n", (unsigned)(benchedSize >> 20)); + srcBuffer = malloc(benchedSize + !benchedSize); + if (!srcBuffer) EXM_THROW(12, "not enough memory"); + + /* Load input buffer */ + BMK_loadFiles(srcBuffer, benchedSize, fileSizes, fileNamesTable, nbFiles); + + /* Bench */ + snprintf (mfName, sizeof(mfName), " %u files", nbFiles); + { const char* displayName = (nbFiles > 1) ? mfName : fileNamesTable[0]; + BMK_benchCLevel(srcBuffer, benchedSize, + displayName, cLevel, cLevelLast, + fileSizes, nbFiles, + dictBuffer, dictBufferSize); + } + + /* clean up */ + free(srcBuffer); + free(dictBuffer); + free(fileSizes); +} + + +static void BMK_syntheticTest(int cLevel, int cLevelLast, double compressibility) +{ + char name[20] = {0}; + size_t benchedSize = 10000000; + void* const srcBuffer = malloc(benchedSize); + + /* Memory allocation */ + if (!srcBuffer) EXM_THROW(21, "not enough memory"); + + /* Fill input buffer */ + RDG_genBuffer(srcBuffer, benchedSize, compressibility, 0.0, 0); + + /* Bench */ + snprintf (name, sizeof(name), "Synthetic %2u%%", (unsigned)(compressibility*100)); + BMK_benchCLevel(srcBuffer, benchedSize, name, cLevel, cLevelLast, &benchedSize, 1, NULL, 0); + + /* clean up */ + free(srcBuffer); +} + + +int BMK_benchFiles(const char** fileNamesTable, unsigned nbFiles, + const char* dictFileName, int cLevel, int cLevelLast) +{ + double const compressibility = (double)g_compressibilityDefault / 100; + + if (nbFiles == 0) + BMK_syntheticTest(cLevel, cLevelLast, compressibility); + else + BMK_benchFileTable(fileNamesTable, nbFiles, dictFileName, cLevel, cLevelLast); + return 0; +} + + + + +/*-************************************ +* Command Line +**************************************/ +static int usage(const char* programName) +{ + DISPLAY(WELCOME_MESSAGE); + DISPLAY( "Usage :\n"); + DISPLAY( " %s [args] [FILE(s)] [-o file]\n", programName); + DISPLAY( "\n"); + DISPLAY( "FILE : a filename\n"); + DISPLAY( " with no FILE, or when FILE is - , read standard input\n"); + DISPLAY( "Arguments :\n"); + DISPLAY( " -D file: use `file` as Dictionary \n"); + DISPLAY( " -h/-H : display help/long help and exit\n"); + DISPLAY( " -V : display Version number and exit\n"); + DISPLAY( " -v : verbose mode; specify multiple times to increase log level (default:%d)\n", DEFAULT_DISPLAY_LEVEL); + DISPLAY( " -q : suppress warnings; specify twice to suppress errors too\n"); +#ifdef UTIL_HAS_CREATEFILELIST + DISPLAY( " -r : operate recursively on directories\n"); +#endif + DISPLAY( "\n"); + DISPLAY( "Benchmark arguments :\n"); + DISPLAY( " -b# : benchmark file(s), using # compression level (default : %d) \n", ZSTDCLI_CLEVEL_DEFAULT); + DISPLAY( " -e# : test all compression levels from -bX to # (default: %d)\n", ZSTDCLI_CLEVEL_DEFAULT); + DISPLAY( " -i# : minimum evaluation time in seconds (default : 3s)\n"); + DISPLAY( " -B# : cut file into independent blocks of size # (default: no block)\n"); + return 0; +} + +static int badusage(const char* programName) +{ + DISPLAYLEVEL(1, "Incorrect parameters\n"); + if (g_displayLevel >= 1) usage(programName); + return 1; +} + +static void waitEnter(void) +{ + int unused; + DISPLAY("Press enter to continue...\n"); + unused = getchar(); + (void)unused; +} + +/*! readU32FromChar() : + @return : unsigned integer value reach from input in `char` format + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : this function can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += (unsigned)(**stringPtr - '0'), (*stringPtr)++ ; + return result; +} + + +#define CLEAN_RETURN(i) { operationResult = (i); goto _end; } + +int main(int argCount, char** argv) +{ + int argNb, + main_pause=0, + nextEntryIsDictionary=0, + operationResult=0, + nextArgumentIsFile=0; + int cLevel = ZSTDCLI_CLEVEL_DEFAULT; + int cLevelLast = 1; + unsigned recursive = 0; + FileNamesTable* filenames = UTIL_allocateFileNamesTable((size_t)argCount); + const char* programName = argv[0]; + const char* dictFileName = NULL; + char* dynNameSpace = NULL; + + /* init */ + if (filenames==NULL) { DISPLAY("zstd: %s \n", strerror(errno)); exit(1); } + displayOut = stderr; + + /* Pick out program name from path. Don't rely on stdlib because of conflicting behavior */ + { size_t pos; + for (pos = strlen(programName); pos > 0; pos--) { if (programName[pos] == '/') { pos++; break; } } + programName += pos; + } + + /* command switches */ + for(argNb=1; argNb<argCount; argNb++) { + const char* argument = argv[argNb]; + if(!argument) continue; /* Protection if argument empty */ + + if (nextArgumentIsFile==0) { + + /* long commands (--long-word) */ + if (!strcmp(argument, "--")) { nextArgumentIsFile=1; continue; } + if (!strcmp(argument, "--version")) { displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); } + if (!strcmp(argument, "--help")) { displayOut=stdout; CLEAN_RETURN(usage(programName)); } + if (!strcmp(argument, "--verbose")) { g_displayLevel++; continue; } + if (!strcmp(argument, "--quiet")) { g_displayLevel--; continue; } + + /* Decode commands (note : aggregated commands are allowed) */ + if (argument[0]=='-') { + argument++; + + while (argument[0]!=0) { + switch(argument[0]) + { + /* Display help */ + case 'V': displayOut=stdout; DISPLAY(WELCOME_MESSAGE); CLEAN_RETURN(0); /* Version Only */ + case 'H': + case 'h': displayOut=stdout; CLEAN_RETURN(usage(programName)); + + /* Use file content as dictionary */ + case 'D': nextEntryIsDictionary = 1; argument++; break; + + /* Verbose mode */ + case 'v': g_displayLevel++; argument++; break; + + /* Quiet mode */ + case 'q': g_displayLevel--; argument++; break; + +#ifdef UTIL_HAS_CREATEFILELIST + /* recursive */ + case 'r': recursive=1; argument++; break; +#endif + + /* Benchmark */ + case 'b': + /* first compression Level */ + argument++; + cLevel = (int)readU32FromChar(&argument); + break; + + /* range bench (benchmark only) */ + case 'e': + /* last compression Level */ + argument++; + cLevelLast = (int)readU32FromChar(&argument); + break; + + /* Modify Nb Iterations (benchmark only) */ + case 'i': + argument++; + { U32 const iters = readU32FromChar(&argument); + BMK_setNotificationLevel(g_displayLevel); + BMK_SetNbIterations(iters); + } + break; + + /* cut input into blocks (benchmark only) */ + case 'B': + argument++; + { size_t bSize = readU32FromChar(&argument); + if (toupper(*argument)=='K') bSize<<=10, argument++; /* allows using KB notation */ + if (toupper(*argument)=='M') bSize<<=20, argument++; + if (toupper(*argument)=='B') argument++; + BMK_setNotificationLevel(g_displayLevel); + BMK_SetBlockSize(bSize); + } + break; + + /* Pause at the end (-p) or set an additional param (-p#) (hidden option) */ + case 'p': argument++; + if ((*argument>='0') && (*argument<='9')) { + BMK_setAdditionalParam((int)readU32FromChar(&argument)); + } else + main_pause=1; + break; + /* unknown command */ + default : CLEAN_RETURN(badusage(programName)); + } + } + continue; + } /* if (argument[0]=='-') */ + + } /* if (nextArgumentIsAFile==0) */ + + if (nextEntryIsDictionary) { + nextEntryIsDictionary = 0; + dictFileName = argument; + continue; + } + + /* add filename to list */ + UTIL_refFilename(filenames, argument); + } + + /* Welcome message (if verbose) */ + DISPLAYLEVEL(3, WELCOME_MESSAGE); + +#ifdef UTIL_HAS_CREATEFILELIST + if (recursive) { + UTIL_expandFNT(&filenames, 1); + } +#endif + + BMK_setNotificationLevel(g_displayLevel); + BMK_benchFiles(filenames->fileNames, (unsigned)filenames->tableSize, dictFileName, cLevel, cLevelLast); + +_end: + if (main_pause) waitEnter(); + free(dynNameSpace); + UTIL_freeFileNamesTable(filenames); + return operationResult; +} diff --git a/src/zstd/zlibWrapper/gzclose.c b/src/zstd/zlibWrapper/gzclose.c new file mode 100644 index 000000000..25d3789b1 --- /dev/null +++ b/src/zstd/zlibWrapper/gzclose.c @@ -0,0 +1,28 @@ +/* gzclose.c contains minimal changes required to be compiled with zlibWrapper: + * - gz_statep was converted to union to work with -Wstrict-aliasing=1 */ + +/* gzclose.c -- zlib gzclose() function + * Copyright (C) 2004, 2010 Mark Adler + * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html + */ + +#include "gzguts.h" + +/* gzclose() is in a separate file so that it is linked in only if it is used. + That way the other gzclose functions can be used instead to avoid linking in + unneeded compression or decompression routines. */ +int ZEXPORT gzclose(file) + gzFile file; +{ +#ifndef NO_GZCOMPRESS + gz_statep state; + + if (file == NULL) + return Z_STREAM_ERROR; + state.file = file; + + return state.state->mode == GZ_READ ? gzclose_r(file) : gzclose_w(file); +#else + return gzclose_r(file); +#endif +} diff --git a/src/zstd/zlibWrapper/gzcompatibility.h b/src/zstd/zlibWrapper/gzcompatibility.h new file mode 100644 index 000000000..394648abb --- /dev/null +++ b/src/zstd/zlibWrapper/gzcompatibility.h @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +#if ZLIB_VERNUM <= 0x1240 +ZEXTERN int ZEXPORT gzclose_r OF((gzFile file)); +ZEXTERN int ZEXPORT gzclose_w OF((gzFile file)); +ZEXTERN int ZEXPORT gzbuffer OF((gzFile file, unsigned size)); +ZEXTERN z_off_t ZEXPORT gzoffset OF((gzFile file)); + +#if !defined(_WIN32) && defined(Z_LARGE64) +# define z_off64_t off64_t +#else +# if defined(_WIN32) && !defined(__GNUC__) && !defined(Z_SOLO) +# define z_off64_t __int64 +# else +# define z_off64_t z_off_t +# endif +#endif +#endif + + +#if ZLIB_VERNUM <= 0x1250 +struct gzFile_s { + unsigned have; + unsigned char *next; + z_off64_t pos; +}; +#endif + + +#if ZLIB_VERNUM <= 0x1270 +#if defined(_WIN32) && !defined(Z_SOLO) +# include <stddef.h> /* for wchar_t */ +ZEXTERN gzFile ZEXPORT gzopen_w OF((const wchar_t *path, + const char *mode)); +#endif +#endif + + +#if ZLIB_VERNUM < 0x12B0 +#ifdef Z_SOLO + typedef unsigned long z_size_t; +#else +# define z_longlong long long +# if defined(NO_SIZE_T) + typedef unsigned NO_SIZE_T z_size_t; +# elif defined(STDC) +# include <stddef.h> + typedef size_t z_size_t; +# else + typedef unsigned long z_size_t; +# endif +# undef z_longlong +#endif +ZEXTERN z_size_t ZEXPORT gzfread OF((voidp buf, z_size_t size, z_size_t nitems, + gzFile file)); +ZEXTERN z_size_t ZEXPORT gzfwrite OF((voidpc buf, z_size_t size, + z_size_t nitems, gzFile file)); +#endif diff --git a/src/zstd/zlibWrapper/gzguts.h b/src/zstd/zlibWrapper/gzguts.h new file mode 100644 index 000000000..b639b4be8 --- /dev/null +++ b/src/zstd/zlibWrapper/gzguts.h @@ -0,0 +1,229 @@ +/* gzguts.h contains minimal changes required to be compiled with zlibWrapper: + * - #include "zlib.h" was changed to #include "zstd_zlibwrapper.h" + * - gz_statep was converted to union to work with -Wstrict-aliasing=1 */ + +/* gzguts.h -- zlib internal header definitions for gz* operations + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler + * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html + */ + +#ifdef _LARGEFILE64_SOURCE +# ifndef _LARGEFILE_SOURCE +# define _LARGEFILE_SOURCE 1 +# endif +# ifdef _FILE_OFFSET_BITS +# undef _FILE_OFFSET_BITS +# endif +#endif + +#ifdef HAVE_HIDDEN +# define ZLIB_INTERNAL __attribute__((visibility ("hidden"))) +#else +# define ZLIB_INTERNAL +#endif + +#include <stdio.h> +#include "zstd_zlibwrapper.h" +#include "gzcompatibility.h" +#ifdef STDC +# include <string.h> +# include <stdlib.h> +# include <limits.h> +#endif + +#ifndef _POSIX_SOURCE +# define _POSIX_SOURCE +#endif +#include <fcntl.h> + +#ifdef _WIN32 +# include <stddef.h> +#else +# include <unistd.h> +#endif + +#if defined(__TURBOC__) || defined(_MSC_VER) || defined(_WIN32) +# include <io.h> +#endif + +#if defined(_WIN32) +# define WIDECHAR +#endif + +#ifdef WINAPI_FAMILY +# define open _open +# define read _read +# define write _write +# define close _close +#endif + +#ifdef NO_DEFLATE /* for compatibility with old definition */ +# define NO_GZCOMPRESS +#endif + +#if defined(STDC99) || (defined(__TURBOC__) && __TURBOC__ >= 0x550) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(__CYGWIN__) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#if defined(MSDOS) && defined(__BORLANDC__) && (BORLANDC > 0x410) +# ifndef HAVE_VSNPRINTF +# define HAVE_VSNPRINTF +# endif +#endif + +#ifndef HAVE_VSNPRINTF +# ifdef MSDOS +/* vsnprintf may exist on some MS-DOS compilers (DJGPP?), + but for now we just assume it doesn't. */ +# define NO_vsnprintf +# endif +# ifdef __TURBOC__ +# define NO_vsnprintf +# endif +# ifdef WIN32 +/* In Win32, vsnprintf is available as the "non-ANSI" _vsnprintf. */ +# if !defined(vsnprintf) && !defined(NO_vsnprintf) +# if !defined(_MSC_VER) || ( defined(_MSC_VER) && _MSC_VER < 1500 ) +# define vsnprintf _vsnprintf +# endif +# endif +# endif +# ifdef __SASC +# define NO_vsnprintf +# endif +# ifdef VMS +# define NO_vsnprintf +# endif +# ifdef __OS400__ +# define NO_vsnprintf +# endif +# ifdef __MVS__ +# define NO_vsnprintf +# endif +#endif + +/* unlike snprintf (which is required in C99), _snprintf does not guarantee + null termination of the result -- however this is only used in gzlib.c where + the result is assured to fit in the space provided */ +#if defined(_MSC_VER) && _MSC_VER < 1900 +# define snprintf _snprintf +#endif + +#ifndef local +# define local static +#endif +/* since "static" is used to mean two completely different things in C, we + define "local" for the non-static meaning of "static", for readability + (compile with -Dlocal if your debugger can't find static symbols) */ + +/* gz* functions always use library allocation functions */ +#ifndef STDC + extern voidp malloc OF((uInt size)); + extern void free OF((voidpf ptr)); +#endif + +/* get errno and strerror definition */ +#if defined UNDER_CE +# include <windows.h> +# define zstrerror() gz_strwinerror((DWORD)GetLastError()) +#else +# ifndef NO_STRERROR +# include <errno.h> +# define zstrerror() strerror(errno) +# else +# define zstrerror() "stdio error (consult errno)" +# endif +#endif + +/* provide prototypes for these when building zlib without LFS */ +#if !defined(_LARGEFILE64_SOURCE) || _LFS64_LARGEFILE-0 == 0 + ZEXTERN gzFile ZEXPORT gzopen64 OF((const char *, const char *)); + ZEXTERN z_off64_t ZEXPORT gzseek64 OF((gzFile, z_off64_t, int)); + ZEXTERN z_off64_t ZEXPORT gztell64 OF((gzFile)); + ZEXTERN z_off64_t ZEXPORT gzoffset64 OF((gzFile)); +#endif + +/* default memLevel */ +#if MAX_MEM_LEVEL >= 8 +# define DEF_MEM_LEVEL 8 +#else +# define DEF_MEM_LEVEL MAX_MEM_LEVEL +#endif + +/* default i/o buffer size -- double this for output when reading (this and + twice this must be able to fit in an unsigned type) */ +#define GZBUFSIZE 8192 + +/* gzip modes, also provide a little integrity check on the passed structure */ +#define GZ_NONE 0 +#define GZ_READ 7247 +#define GZ_WRITE 31153 +#define GZ_APPEND 1 /* mode set to GZ_WRITE after the file is opened */ + +/* values for gz_state how */ +#define LOOK 0 /* look for a gzip header */ +#define COPY 1 /* copy input directly */ +#define GZIP 2 /* decompress a gzip stream */ + +/* internal gzip file state data structure */ +typedef struct { + /* exposed contents for gzgetc() macro */ + struct gzFile_s x; /* "x" for exposed */ + /* x.have: number of bytes available at x.next */ + /* x.next: next output data to deliver or write */ + /* x.pos: current position in uncompressed data */ + /* used for both reading and writing */ + int mode; /* see gzip modes above */ + int fd; /* file descriptor */ + char *path; /* path or fd for error messages */ + unsigned size; /* buffer size, zero if not allocated yet */ + unsigned want; /* requested buffer size, default is GZBUFSIZE */ + unsigned char *in; /* input buffer (double-sized when writing) */ + unsigned char *out; /* output buffer (double-sized when reading) */ + int direct; /* 0 if processing gzip, 1 if transparent */ + /* just for reading */ + int how; /* 0: get header, 1: copy, 2: decompress */ + z_off64_t start; /* where the gzip data started, for rewinding */ + int eof; /* true if end of input file reached */ + int past; /* true if read requested past end */ + /* just for writing */ + int level; /* compression level */ + int strategy; /* compression strategy */ + /* seek request */ + z_off64_t skip; /* amount to skip (already rewound if backwards) */ + int seek; /* true if seek request pending */ + /* error information */ + int err; /* error code */ + char *msg; /* error message */ + /* zlib inflate or deflate stream */ + z_stream strm; /* stream structure in-place (not a pointer) */ +} gz_state; + +typedef union { + gz_state FAR *state; + gzFile file; +} gz_statep; + +/* shared functions */ +void ZLIB_INTERNAL gz_error OF((gz_statep, int, const char *)); +#if defined UNDER_CE +char ZLIB_INTERNAL *gz_strwinerror OF((DWORD error)); +#endif + +/* GT_OFF(x), where x is an unsigned value, is true if x > maximum z_off64_t + value -- needed when comparing unsigned to z_off64_t, which is signed + (possible z_off64_t types off_t, off64_t, and long are all signed) */ +#ifdef INT_MAX +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > INT_MAX) +#else +unsigned ZLIB_INTERNAL gz_intmax OF((void)); +# define GT_OFF(x) (sizeof(int) == sizeof(z_off64_t) && (x) > gz_intmax()) +#endif diff --git a/src/zstd/zlibWrapper/gzlib.c b/src/zstd/zlibWrapper/gzlib.c new file mode 100644 index 000000000..b1fb98517 --- /dev/null +++ b/src/zstd/zlibWrapper/gzlib.c @@ -0,0 +1,640 @@ +/* gzlib.c contains minimal changes required to be compiled with zlibWrapper: + * - gz_statep was converted to union to work with -Wstrict-aliasing=1 */ + +/* gzlib.c -- zlib functions common to reading and writing gzip files + * Copyright (C) 2004-2017 Mark Adler + * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html + */ + +#include "gzguts.h" + +#if defined(_WIN32) && !defined(__BORLANDC__) && !defined(__MINGW32__) +# define LSEEK _lseeki64 +#else +#if defined(_LARGEFILE64_SOURCE) && _LFS64_LARGEFILE-0 +# define LSEEK lseek64 +#else +# define LSEEK lseek +#endif +#endif + +/* Local functions */ +local void gz_reset OF((gz_statep)); +local gzFile gz_open OF((const void *, int, const char *)); + +#if defined UNDER_CE + +/* Map the Windows error number in ERROR to a locale-dependent error message + string and return a pointer to it. Typically, the values for ERROR come + from GetLastError. + + The string pointed to shall not be modified by the application, but may be + overwritten by a subsequent call to gz_strwinerror + + The gz_strwinerror function does not change the current setting of + GetLastError. */ +char ZLIB_INTERNAL *gz_strwinerror (error) + DWORD error; +{ + static char buf[1024]; + + wchar_t *msgbuf; + DWORD lasterr = GetLastError(); + DWORD chars = FormatMessage(FORMAT_MESSAGE_FROM_SYSTEM + | FORMAT_MESSAGE_ALLOCATE_BUFFER, + NULL, + error, + 0, /* Default language */ + (LPVOID)&msgbuf, + 0, + NULL); + if (chars != 0) { + /* If there is an \r\n appended, zap it. */ + if (chars >= 2 + && msgbuf[chars - 2] == '\r' && msgbuf[chars - 1] == '\n') { + chars -= 2; + msgbuf[chars] = 0; + } + + if (chars > sizeof (buf) - 1) { + chars = sizeof (buf) - 1; + msgbuf[chars] = 0; + } + + wcstombs(buf, msgbuf, chars + 1); + LocalFree(msgbuf); + } + else { + sprintf(buf, "unknown win32 error (%ld)", error); + } + + SetLastError(lasterr); + return buf; +} + +#endif /* UNDER_CE */ + +/* Reset gzip file state */ +local void gz_reset(state) + gz_statep state; +{ + state.state->x.have = 0; /* no output data available */ + if (state.state->mode == GZ_READ) { /* for reading ... */ + state.state->eof = 0; /* not at end of file */ + state.state->past = 0; /* have not read past end yet */ + state.state->how = LOOK; /* look for gzip header */ + } + state.state->seek = 0; /* no seek request pending */ + gz_error(state, Z_OK, NULL); /* clear error */ + state.state->x.pos = 0; /* no uncompressed data yet */ + state.state->strm.avail_in = 0; /* no input data yet */ +} + +/* Open a gzip file either by name or file descriptor. */ +local gzFile gz_open(path, fd, mode) + const void *path; + int fd; + const char *mode; +{ + gz_statep state; + z_size_t len; + int oflag; +#ifdef O_CLOEXEC + int cloexec = 0; +#endif +#ifdef O_EXCL + int exclusive = 0; +#endif + + /* check input */ + if (path == NULL) + return NULL; + + /* allocate gzFile structure to return */ + state.state = (gz_state*)malloc(sizeof(gz_state)); + if (state.state == NULL) + return NULL; + state.state->size = 0; /* no buffers allocated yet */ + state.state->want = GZBUFSIZE; /* requested buffer size */ + state.state->msg = NULL; /* no error message yet */ + + /* interpret mode */ + state.state->mode = GZ_NONE; + state.state->level = Z_DEFAULT_COMPRESSION; + state.state->strategy = Z_DEFAULT_STRATEGY; + state.state->direct = 0; + while (*mode) { + if (*mode >= '0' && *mode <= '9') + state.state->level = *mode - '0'; + else + switch (*mode) { + case 'r': + state.state->mode = GZ_READ; + break; +#ifndef NO_GZCOMPRESS + case 'w': + state.state->mode = GZ_WRITE; + break; + case 'a': + state.state->mode = GZ_APPEND; + break; +#endif + case '+': /* can't read and write at the same time */ + free(state.state); + return NULL; + case 'b': /* ignore -- will request binary anyway */ + break; +#ifdef O_CLOEXEC + case 'e': + cloexec = 1; + break; +#endif +#ifdef O_EXCL + case 'x': + exclusive = 1; + break; +#endif + case 'f': + state.state->strategy = Z_FILTERED; + break; + case 'h': + state.state->strategy = Z_HUFFMAN_ONLY; + break; + case 'R': + state.state->strategy = Z_RLE; + break; + case 'F': + state.state->strategy = Z_FIXED; + break; + case 'T': + state.state->direct = 1; + break; + default: /* could consider as an error, but just ignore */ + ; + } + mode++; + } + + /* must provide an "r", "w", or "a" */ + if (state.state->mode == GZ_NONE) { + free(state.state); + return NULL; + } + + /* can't force transparent read */ + if (state.state->mode == GZ_READ) { + if (state.state->direct) { + free(state.state); + return NULL; + } + state.state->direct = 1; /* for empty file */ + } + + /* save the path name for error messages */ +#ifdef WIDECHAR + if (fd == -2) { + len = wcstombs(NULL, path, 0); + if (len == (z_size_t)-1) + len = 0; + } + else +#endif + len = strlen((const char *)path); + state.state->path = (char *)malloc(len + 1); + if (state.state->path == NULL) { + free(state.state); + return NULL; + } +#ifdef WIDECHAR + if (fd == -2) + if (len) + wcstombs(state.state->path, path, len + 1); + else + *(state.state->path) = 0; + else +#endif +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(state.state->path, len + 1, "%s", (const char *)path); +#else + strcpy(state.state->path, (const char*)path); +#endif + + /* compute the flags for open() */ + oflag = +#ifdef O_LARGEFILE + O_LARGEFILE | +#endif +#ifdef O_BINARY + O_BINARY | +#endif +#ifdef O_CLOEXEC + (cloexec ? O_CLOEXEC : 0) | +#endif + (state.state->mode == GZ_READ ? + O_RDONLY : + (O_WRONLY | O_CREAT | +#ifdef O_EXCL + (exclusive ? O_EXCL : 0) | +#endif + (state.state->mode == GZ_WRITE ? + O_TRUNC : + O_APPEND))); + + /* open the file with the appropriate flags (or just use fd) */ + state.state->fd = fd > -1 ? fd : ( +#ifdef WIDECHAR + fd == -2 ? _wopen(path, oflag, 0666) : +#endif + open((const char *)path, oflag, 0666)); + if (state.state->fd == -1) { + free(state.state->path); + free(state.state); + return NULL; + } + if (state.state->mode == GZ_APPEND) { + LSEEK(state.state->fd, 0, SEEK_END); /* so gzoffset() is correct */ + state.state->mode = GZ_WRITE; /* simplify later checks */ + } + + /* save the current position for rewinding (only if reading) */ + if (state.state->mode == GZ_READ) { + state.state->start = LSEEK(state.state->fd, 0, SEEK_CUR); + if (state.state->start == -1) state.state->start = 0; + } + + /* initialize stream */ + gz_reset(state); + + /* return stream */ + return state.file; +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzopen(path, mode) + const char *path; + const char *mode; +{ + return gz_open(path, -1, mode); +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzopen64(path, mode) + const char *path; + const char *mode; +{ + return gz_open(path, -1, mode); +} + +/* -- see zlib.h -- */ +gzFile ZEXPORT gzdopen(fd, mode) + int fd; + const char *mode; +{ + char *path; /* identifier for error messages */ + gzFile gz; + + if (fd == -1 || (path = (char *)malloc(7 + 3 * sizeof(int))) == NULL) + return NULL; +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(path, 7 + 3 * sizeof(int), "<fd:%d>", fd); +#else + sprintf(path, "<fd:%d>", fd); /* for debugging */ +#endif + gz = gz_open(path, fd, mode); + free(path); + return gz; +} + +/* -- see zlib.h -- */ +#ifdef WIDECHAR +gzFile ZEXPORT gzopen_w(path, mode) + const wchar_t *path; + const char *mode; +{ + return gz_open(path, -2, mode); +} +#endif + +/* -- see zlib.h -- */ +int ZEXPORT gzbuffer(file, size) + gzFile file; + unsigned size; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state.file = file; + if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) + return -1; + + /* make sure we haven't already allocated memory */ + if (state.state->size != 0) + return -1; + + /* check and set requested size */ + if ((size << 1) < size) + return -1; /* need to be able to double it */ + if (size < 2) + size = 2; /* need two bytes to check magic header */ + state.state->want = size; + return 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzrewind(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state.file = file; + + /* check that we're reading and that there's no error */ + if (state.state->mode != GZ_READ || + (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR)) + return -1; + + /* back up and start over */ + if (LSEEK(state.state->fd, state.state->start, SEEK_SET) == -1) + return -1; + gz_reset(state); + return 0; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gzseek64(file, offset, whence) + gzFile file; + z_off64_t offset; + int whence; +{ + unsigned n; + z_off64_t ret; + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state.file = file; + if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) + return -1; + + /* check that there's no error */ + if (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR) + return -1; + + /* can only seek from start or relative to current position */ + if (whence != SEEK_SET && whence != SEEK_CUR) + return -1; + + /* normalize offset to a SEEK_CUR specification */ + if (whence == SEEK_SET) + offset -= state.state->x.pos; + else if (state.state->seek) + offset += state.state->skip; + state.state->seek = 0; + + /* if within raw area while reading, just go there */ + if (state.state->mode == GZ_READ && state.state->how == COPY && + state.state->x.pos + offset >= 0) { + ret = LSEEK(state.state->fd, offset - state.state->x.have, SEEK_CUR); + if (ret == -1) + return -1; + state.state->x.have = 0; + state.state->eof = 0; + state.state->past = 0; + state.state->seek = 0; + gz_error(state, Z_OK, NULL); + state.state->strm.avail_in = 0; + state.state->x.pos += offset; + return state.state->x.pos; + } + + /* calculate skip amount, rewinding if needed for back seek when reading */ + if (offset < 0) { + if (state.state->mode != GZ_READ) /* writing -- can't go backwards */ + return -1; + offset += state.state->x.pos; + if (offset < 0) /* before start of file! */ + return -1; + if (gzrewind(file) == -1) /* rewind, then skip to offset */ + return -1; + } + + /* if reading, skip what's in output buffer (one less gzgetc() check) */ + if (state.state->mode == GZ_READ) { + n = GT_OFF(state.state->x.have) || (z_off64_t)state.state->x.have > offset ? + (unsigned)offset : state.state->x.have; + state.state->x.have -= n; + state.state->x.next += n; + state.state->x.pos += n; + offset -= n; + } + + /* request skip (if not zero) */ + if (offset) { + state.state->seek = 1; + state.state->skip = offset; + } + return state.state->x.pos + offset; +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gzseek(file, offset, whence) + gzFile file; + z_off_t offset; + int whence; +{ + z_off64_t ret; + + ret = gzseek64(file, (z_off64_t)offset, whence); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gztell64(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state.file = file; + if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) + return -1; + + /* return position */ + return state.state->x.pos + (state.state->seek ? state.state->skip : 0); +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gztell(file) + gzFile file; +{ + z_off64_t ret; + + ret = gztell64(file); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +z_off64_t ZEXPORT gzoffset64(file) + gzFile file; +{ + z_off64_t offset; + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return -1; + state.file = file; + if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) + return -1; + + /* compute and return effective offset in file */ + offset = LSEEK(state.state->fd, 0, SEEK_CUR); + if (offset == -1) + return -1; + if (state.state->mode == GZ_READ) /* reading */ + offset -= state.state->strm.avail_in; /* don't count buffered input */ + return offset; +} + +/* -- see zlib.h -- */ +z_off_t ZEXPORT gzoffset(file) + gzFile file; +{ + z_off64_t ret; + + ret = gzoffset64(file); + return ret == (z_off_t)ret ? (z_off_t)ret : -1; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzeof(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return 0; + state.file = file; + if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) + return 0; + + /* return end-of-file state */ + return state.state->mode == GZ_READ ? state.state->past : 0; +} + +/* -- see zlib.h -- */ +const char * ZEXPORT gzerror(file, errnum) + gzFile file; + int *errnum; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return NULL; + state.file = file; + if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) + return NULL; + + /* return error information */ + if (errnum != NULL) + *errnum = state.state->err; + return state.state->err == Z_MEM_ERROR ? "out of memory" : + (state.state->msg == NULL ? "" : state.state->msg); +} + +/* -- see zlib.h -- */ +void ZEXPORT gzclearerr(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure and check integrity */ + if (file == NULL) + return; + state.file = file; + if (state.state->mode != GZ_READ && state.state->mode != GZ_WRITE) + return; + + /* clear error and end-of-file */ + if (state.state->mode == GZ_READ) { + state.state->eof = 0; + state.state->past = 0; + } + gz_error(state, Z_OK, NULL); +} + +/* Create an error message in allocated memory and set state.state->err and + state.state->msg accordingly. Free any previous error message already there. Do + not try to free or allocate space if the error is Z_MEM_ERROR (out of + memory). Simply save the error message as a static string. If there is an + allocation failure constructing the error message, then convert the error to + out of memory. */ +void ZLIB_INTERNAL gz_error(state, err, msg) + gz_statep state; + int err; + const char *msg; +{ + /* free previously allocated message and clear */ + if (state.state->msg != NULL) { + if (state.state->err != Z_MEM_ERROR) + free(state.state->msg); + state.state->msg = NULL; + } + + /* if fatal, set state.state->x.have to 0 so that the gzgetc() macro fails */ + if (err != Z_OK && err != Z_BUF_ERROR) + state.state->x.have = 0; + + /* set error code, and if no message, then done */ + state.state->err = err; + if (msg == NULL) + return; + + /* for an out of memory error, return literal string when requested */ + if (err == Z_MEM_ERROR) + return; + + /* construct error message with path */ + if ((state.state->msg = (char *)malloc(strlen(state.state->path) + strlen(msg) + 3)) == + NULL) { + state.state->err = Z_MEM_ERROR; + return; + } +#if !defined(NO_snprintf) && !defined(NO_vsnprintf) + (void)snprintf(state.state->msg, strlen(state.state->path) + strlen(msg) + 3, + "%s%s%s", state.state->path, ": ", msg); +#else + strcpy(state.state->msg, state.state->path); + strcat(state.state->msg, ": "); + strcat(state.state->msg, msg); +#endif +} + +#ifndef INT_MAX +/* portably return maximum value for an int (when limits.h presumed not + available) -- we need to do this to cover cases where 2's complement not + used, since C standard permits 1's complement and sign-bit representations, + otherwise we could just use ((unsigned)-1) >> 1 */ +unsigned ZLIB_INTERNAL gz_intmax() +{ + unsigned p, q; + + p = 1; + do { + q = p; + p <<= 1; + p++; + } while (p > q); + return q >> 1; +} +#endif diff --git a/src/zstd/zlibWrapper/gzread.c b/src/zstd/zlibWrapper/gzread.c new file mode 100644 index 000000000..359d17889 --- /dev/null +++ b/src/zstd/zlibWrapper/gzread.c @@ -0,0 +1,678 @@ +/* gzread.c contains minimal changes required to be compiled with zlibWrapper: + * - gz_statep was converted to union to work with -Wstrict-aliasing=1 */ + + /* gzread.c -- zlib functions for reading gzip files + * Copyright (C) 2004, 2005, 2010, 2011, 2012, 2013, 2016 Mark Adler + * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html + */ + +#include "gzguts.h" + +/* fix for Visual Studio, which doesn't support ssize_t type. + * see https://github.com/facebook/zstd/issues/1800#issuecomment-545945050 */ +#if defined(_MSC_VER) && !defined(ssize_t) +# include <BaseTsd.h> + typedef SSIZE_T ssize_t; +#endif + + +/* Local functions */ +local int gz_load OF((gz_statep, unsigned char *, unsigned, unsigned *)); +local int gz_avail OF((gz_statep)); +local int gz_look OF((gz_statep)); +local int gz_decomp OF((gz_statep)); +local int gz_fetch OF((gz_statep)); +local int gz_skip OF((gz_statep, z_off64_t)); +local z_size_t gz_read OF((gz_statep, voidp, z_size_t)); + +/* Use read() to load a buffer -- return -1 on error, otherwise 0. Read from + state.state->fd, and update state.state->eof, state.state->err, and state.state->msg as appropriate. + This function needs to loop on read(), since read() is not guaranteed to + read the number of bytes requested, depending on the type of descriptor. */ +local int gz_load(state, buf, len, have) + gz_statep state; + unsigned char *buf; + unsigned len; + unsigned *have; +{ + ssize_t ret; + unsigned get, max = ((unsigned)-1 >> 2) + 1; + + *have = 0; + do { + get = len - *have; + if (get > max) + get = max; + ret = read(state.state->fd, buf + *have, get); + if (ret <= 0) + break; + *have += (unsigned)ret; + } while (*have < len); + if (ret < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + if (ret == 0) + state.state->eof = 1; + return 0; +} + +/* Load up input buffer and set eof flag if last data loaded -- return -1 on + error, 0 otherwise. Note that the eof flag is set when the end of the input + file is reached, even though there may be unused data in the buffer. Once + that data has been used, no more attempts will be made to read the file. + If strm->avail_in != 0, then the current data is moved to the beginning of + the input buffer, and then the remainder of the buffer is loaded with the + available data from the input file. */ +local int gz_avail(state) + gz_statep state; +{ + unsigned got; + z_streamp strm = &(state.state->strm); + + if (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR) + return -1; + if (state.state->eof == 0) { + if (strm->avail_in) { /* copy what's there to the start */ + unsigned char *p = state.state->in; + unsigned const char *q = strm->next_in; + unsigned n = strm->avail_in; + do { + *p++ = *q++; + } while (--n); + } + if (gz_load(state, state.state->in + strm->avail_in, + state.state->size - strm->avail_in, &got) == -1) + return -1; + strm->avail_in += got; + strm->next_in = state.state->in; + } + return 0; +} + +/* Look for gzip header, set up for inflate or copy. state.state->x.have must be 0. + If this is the first time in, allocate required memory. state.state->how will be + left unchanged if there is no more input data available, will be set to COPY + if there is no gzip header and direct copying will be performed, or it will + be set to GZIP for decompression. If direct copying, then leftover input + data from the input buffer will be copied to the output buffer. In that + case, all further file reads will be directly to either the output buffer or + a user buffer. If decompressing, the inflate state will be initialized. + gz_look() will return 0 on success or -1 on failure. */ +local int gz_look(state) + gz_statep state; +{ + z_streamp strm = &(state.state->strm); + + /* allocate read buffers and inflate memory */ + if (state.state->size == 0) { + /* allocate buffers */ + state.state->in = (unsigned char *)malloc(state.state->want); + state.state->out = (unsigned char *)malloc(state.state->want << 1); + if (state.state->in == NULL || state.state->out == NULL) { + free(state.state->out); + free(state.state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + state.state->size = state.state->want; + + /* allocate inflate memory */ + state.state->strm.zalloc = Z_NULL; + state.state->strm.zfree = Z_NULL; + state.state->strm.opaque = Z_NULL; + state.state->strm.avail_in = 0; + state.state->strm.next_in = Z_NULL; + if (inflateInit2(&(state.state->strm), 15 + 16) != Z_OK) { /* gunzip */ + free(state.state->out); + free(state.state->in); + state.state->size = 0; + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + } + + /* get at least the magic bytes in the input buffer */ + if (strm->avail_in < 2) { + if (gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0) + return 0; + } + + /* look for gzip magic bytes -- if there, do gzip decoding (note: there is + a logical dilemma here when considering the case of a partially written + gzip file, to wit, if a single 31 byte is written, then we cannot tell + whether this is a single-byte file, or just a partially written gzip + file -- for here we assume that if a gzip file is being written, then + the header will be written in a single operation, so that reading a + single byte is sufficient indication that it is not a gzip file) */ + if (strm->avail_in > 1 && + ((strm->next_in[0] == 31 && strm->next_in[1] == 139) /* gz header */ + || (strm->next_in[0] == 40 && strm->next_in[1] == 181))) { /* zstd header */ + inflateReset(strm); + state.state->how = GZIP; + state.state->direct = 0; + return 0; + } + + /* no gzip header -- if we were decoding gzip before, then this is trailing + garbage. Ignore the trailing garbage and finish. */ + if (state.state->direct == 0) { + strm->avail_in = 0; + state.state->eof = 1; + state.state->x.have = 0; + return 0; + } + + /* doing raw i/o, copy any leftover input to output -- this assumes that + the output buffer is larger than the input buffer, which also assures + space for gzungetc() */ + state.state->x.next = state.state->out; + if (strm->avail_in) { + memcpy(state.state->x.next, strm->next_in, strm->avail_in); + state.state->x.have = strm->avail_in; + strm->avail_in = 0; + } + state.state->how = COPY; + state.state->direct = 1; + return 0; +} + +/* Decompress from input to the provided next_out and avail_out in the state. + On return, state.state->x.have and state.state->x.next point to the just decompressed + data. If the gzip stream completes, state.state->how is reset to LOOK to look for + the next gzip stream or raw data, once state.state->x.have is depleted. Returns 0 + on success, -1 on failure. */ +local int gz_decomp(state) + gz_statep state; +{ + int ret = Z_OK; + unsigned had; + z_streamp strm = &(state.state->strm); + + /* fill output buffer up to end of deflate stream */ + had = strm->avail_out; + do { + /* get more input for inflate() */ + if (strm->avail_in == 0 && gz_avail(state) == -1) + return -1; + if (strm->avail_in == 0) { + gz_error(state, Z_BUF_ERROR, "unexpected end of file"); + break; + } + + /* decompress and handle errors */ + ret = inflate(strm, Z_NO_FLUSH); + if (ret == Z_STREAM_ERROR || ret == Z_NEED_DICT) { + gz_error(state, Z_STREAM_ERROR, + "internal error: inflate stream corrupt"); + return -1; + } + if (ret == Z_MEM_ERROR) { + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + if (ret == Z_DATA_ERROR) { /* deflate stream invalid */ + gz_error(state, Z_DATA_ERROR, + strm->msg == NULL ? "compressed data error" : strm->msg); + return -1; + } + } while (strm->avail_out && ret != Z_STREAM_END); + + /* update available output */ + state.state->x.have = had - strm->avail_out; + state.state->x.next = strm->next_out - state.state->x.have; + + /* if the gzip stream completed successfully, look for another */ + if (ret == Z_STREAM_END) + state.state->how = LOOK; + + /* good decompression */ + return 0; +} + +/* Fetch data and put it in the output buffer. Assumes state.state->x.have is 0. + Data is either copied from the input file or decompressed from the input + file depending on state.state->how. If state.state->how is LOOK, then a gzip header is + looked for to determine whether to copy or decompress. Returns -1 on error, + otherwise 0. gz_fetch() will leave state.state->how as COPY or GZIP unless the + end of the input file has been reached and all data has been processed. */ +local int gz_fetch(state) + gz_statep state; +{ + z_streamp strm = &(state.state->strm); + + do { + switch(state.state->how) { + case LOOK: /* -> LOOK, COPY (only if never GZIP), or GZIP */ + if (gz_look(state) == -1) + return -1; + if (state.state->how == LOOK) + return 0; + break; + case COPY: /* -> COPY */ + if (gz_load(state, state.state->out, state.state->size << 1, &(state.state->x.have)) + == -1) + return -1; + state.state->x.next = state.state->out; + return 0; + case GZIP: /* -> GZIP or LOOK (if end of gzip stream) */ + strm->avail_out = state.state->size << 1; + strm->next_out = state.state->out; + if (gz_decomp(state) == -1) + return -1; + } + } while (state.state->x.have == 0 && (!state.state->eof || strm->avail_in)); + return 0; +} + +/* Skip len uncompressed bytes of output. Return -1 on error, 0 on success. */ +local int gz_skip(state, len) + gz_statep state; + z_off64_t len; +{ + unsigned n; + + /* skip over len bytes or reach end-of-file, whichever comes first */ + while (len) + /* skip over whatever is in output buffer */ + if (state.state->x.have) { + n = GT_OFF(state.state->x.have) || (z_off64_t)state.state->x.have > len ? + (unsigned)len : state.state->x.have; + state.state->x.have -= n; + state.state->x.next += n; + state.state->x.pos += n; + len -= n; + } + + /* output buffer empty -- return if we're at the end of the input */ + else if (state.state->eof && state.state->strm.avail_in == 0) + break; + + /* need more data to skip -- load up output buffer */ + else { + /* get more output, looking for header if required */ + if (gz_fetch(state) == -1) + return -1; + } + return 0; +} + +/* Read len bytes into buf from file, or less than len up to the end of the + input. Return the number of bytes read. If zero is returned, either the + end of file was reached, or there was an error. state.state->err must be + consulted in that case to determine which. */ +local z_size_t gz_read(state, buf, len) + gz_statep state; + voidp buf; + z_size_t len; +{ + z_size_t got; + unsigned n; + + /* if len is zero, avoid unnecessary operations */ + if (len == 0) + return 0; + + /* process a skip request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_skip(state, state.state->skip) == -1) + return 0; + } + + /* get len bytes to buf, or less than len if at the end */ + got = 0; + do { + /* set n to the maximum amount of len that fits in an unsigned int */ + n = -1; + if (n > len) + n = (unsigned)len; + + /* first just try copying data from the output buffer */ + if (state.state->x.have) { + if (state.state->x.have < n) + n = state.state->x.have; + memcpy(buf, state.state->x.next, n); + state.state->x.next += n; + state.state->x.have -= n; + } + + /* output buffer empty -- return if we're at the end of the input */ + else if (state.state->eof && state.state->strm.avail_in == 0) { + state.state->past = 1; /* tried to read past end */ + break; + } + + /* need output data -- for small len or new stream load up our output + buffer */ + else if (state.state->how == LOOK || n < (state.state->size << 1)) { + /* get more output, looking for header if required */ + if (gz_fetch(state) == -1) + return 0; + continue; /* no progress yet -- go back to copy above */ + /* the copy above assures that we will leave with space in the + output buffer, allowing at least one gzungetc() to succeed */ + } + + /* large len -- read directly into user buffer */ + else if (state.state->how == COPY) { /* read directly */ + if (gz_load(state, (unsigned char *)buf, n, &n) == -1) + return 0; + } + + /* large len -- decompress directly into user buffer */ + else { /* state.state->how == GZIP */ + state.state->strm.avail_out = n; + state.state->strm.next_out = (unsigned char *)buf; + if (gz_decomp(state) == -1) + return 0; + n = state.state->x.have; + state.state->x.have = 0; + } + + /* update progress */ + len -= n; + buf = (char *)buf + n; + got += n; + state.state->x.pos += n; + } while (len); + + /* return number of bytes read into user buffer */ + return got; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzread(file, buf, len) + gzFile file; + voidp buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state.file = file; + + /* check that we're reading and that there's no (serious) error */ + if (state.state->mode != GZ_READ || + (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR)) + return -1; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in an int"); + return -1; + } + + /* read len or fewer bytes to buf */ + len = (unsigned)gz_read(state, buf, len); + + /* check for an error */ + if (len == 0 && state.state->err != Z_OK && state.state->err != Z_BUF_ERROR) + return -1; + + /* return the number of bytes read (this is assured to fit in an int) */ + return (int)len; +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfread(buf, size, nitems, file) + voidp buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state.file = file; + + /* check that we're reading and that there's no (serious) error */ + if (state.state->mode != GZ_READ || + (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR)) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && len / size != nitems) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* read len or fewer bytes to buf, return the number of full items read */ + return len ? gz_read(state, buf, len) / size : 0; +} + +/* -- see zlib.h -- */ +#if ZLIB_VERNUM >= 0x1261 +#ifdef Z_PREFIX_SET +# undef z_gzgetc +#else +# undef gzgetc +#endif +#endif + +#if ZLIB_VERNUM == 0x1260 +# undef gzgetc +#endif + +#if ZLIB_VERNUM <= 0x1250 +ZEXTERN int ZEXPORT gzgetc OF((gzFile file)); +ZEXTERN int ZEXPORT gzgetc_ OF((gzFile file)); +#endif + +int ZEXPORT gzgetc(file) + gzFile file; +{ + int ret; + unsigned char buf[1]; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state.file = file; + + /* check that we're reading and that there's no (serious) error */ + if (state.state->mode != GZ_READ || + (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR)) + return -1; + + /* try output buffer (no need to check for skip request) */ + if (state.state->x.have) { + state.state->x.have--; + state.state->x.pos++; + return *(state.state->x.next)++; + } + + /* nothing there -- try gz_read() */ + ret = (int)gz_read(state, buf, 1); + return ret < 1 ? -1 : buf[0]; +} + +int ZEXPORT gzgetc_(file) +gzFile file; +{ + return gzgetc(file); +} + +/* -- see zlib.h -- */ +int ZEXPORT gzungetc(c, file) + int c; + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state.file = file; + + /* check that we're reading and that there's no (serious) error */ + if (state.state->mode != GZ_READ || + (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR)) + return -1; + + /* process a skip request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_skip(state, state.state->skip) == -1) + return -1; + } + + /* can't push EOF */ + if (c < 0) + return -1; + + /* if output buffer empty, put byte at end (allows more pushing) */ + if (state.state->x.have == 0) { + state.state->x.have = 1; + state.state->x.next = state.state->out + (state.state->size << 1) - 1; + state.state->x.next[0] = (unsigned char)c; + state.state->x.pos--; + state.state->past = 0; + return c; + } + + /* if no room, give up (must have already done a gzungetc()) */ + if (state.state->x.have == (state.state->size << 1)) { + gz_error(state, Z_DATA_ERROR, "out of room to push characters"); + return -1; + } + + /* slide output data if needed and insert byte before existing data */ + if (state.state->x.next == state.state->out) { + unsigned char *src = state.state->out + state.state->x.have; + unsigned char *dest = state.state->out + (state.state->size << 1); + while (src > state.state->out) + *--dest = *--src; + state.state->x.next = dest; + } + state.state->x.have++; + state.state->x.next--; + state.state->x.next[0] = (unsigned char)c; + state.state->x.pos--; + state.state->past = 0; + return c; +} + +/* -- see zlib.h -- */ +char * ZEXPORT gzgets(file, buf, len) + gzFile file; + char *buf; + int len; +{ + unsigned left, n; + char *str; + unsigned char *eol; + gz_statep state; + + /* check parameters and get internal structure */ + if (file == NULL || buf == NULL || len < 1) + return NULL; + state.file = file; + + /* check that we're reading and that there's no (serious) error */ + if (state.state->mode != GZ_READ || + (state.state->err != Z_OK && state.state->err != Z_BUF_ERROR)) + return NULL; + + /* process a skip request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_skip(state, state.state->skip) == -1) + return NULL; + } + + /* copy output bytes up to new line or len - 1, whichever comes first -- + append a terminating zero to the string (we don't check for a zero in + the contents, let the user worry about that) */ + str = buf; + left = (unsigned)len - 1; + if (left) do { + /* assure that something is in the output buffer */ + if (state.state->x.have == 0 && gz_fetch(state) == -1) + return NULL; /* error */ + if (state.state->x.have == 0) { /* end of file */ + state.state->past = 1; /* read past end */ + break; /* return what we have */ + } + + /* look for end-of-line in current output buffer */ + n = state.state->x.have > left ? left : state.state->x.have; + eol = (unsigned char *)memchr(state.state->x.next, '\n', n); + if (eol != NULL) + n = (unsigned)(eol - state.state->x.next) + 1; + + /* copy through end-of-line, or remainder if not found */ + memcpy(buf, state.state->x.next, n); + state.state->x.have -= n; + state.state->x.next += n; + state.state->x.pos += n; + left -= n; + buf += n; + } while (left && eol == NULL); + + /* return terminated string, or if nothing, end of file */ + if (buf == str) + return NULL; + buf[0] = 0; + return str; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzdirect(file) + gzFile file; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state.file = file; + + /* if the state is not known, but we can find out, then do so (this is + mainly for right after a gzopen() or gzdopen()) */ + if (state.state->mode == GZ_READ && state.state->how == LOOK && state.state->x.have == 0) + (void)gz_look(state); + + /* return 1 if transparent, 0 if processing a gzip stream */ + return state.state->direct; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzclose_r(file) + gzFile file; +{ + int ret, err; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state.file = file; + + /* check that we're reading */ + if (state.state->mode != GZ_READ) + return Z_STREAM_ERROR; + + /* free memory and close file */ + if (state.state->size) { + inflateEnd(&(state.state->strm)); + free(state.state->out); + free(state.state->in); + } + err = state.state->err == Z_BUF_ERROR ? Z_BUF_ERROR : Z_OK; + gz_error(state, Z_OK, NULL); + free(state.state->path); + ret = close(state.state->fd); + free(state.state); + return ret ? Z_ERRNO : err; +} diff --git a/src/zstd/zlibWrapper/gzwrite.c b/src/zstd/zlibWrapper/gzwrite.c new file mode 100644 index 000000000..422ff17db --- /dev/null +++ b/src/zstd/zlibWrapper/gzwrite.c @@ -0,0 +1,671 @@ +/* gzwrite.c contains minimal changes required to be compiled with zlibWrapper: + * - gz_statep was converted to union to work with -Wstrict-aliasing=1 */ + + /* gzwrite.c -- zlib functions for writing gzip files + * Copyright (C) 2004-2017 Mark Adler + * For conditions of distribution and use, see http://www.zlib.net/zlib_license.html + */ + +#include <assert.h> + +#include "gzguts.h" + +/* Local functions */ +local int gz_init OF((gz_statep)); +local int gz_comp OF((gz_statep, int)); +local int gz_zero OF((gz_statep, z_off64_t)); +local z_size_t gz_write OF((gz_statep, voidpc, z_size_t)); + +/* Initialize state for writing a gzip file. Mark initialization by setting + state.state->size to non-zero. Return -1 on a memory allocation failure, or 0 on + success. */ +local int gz_init(state) + gz_statep state; +{ + int ret; + z_streamp strm = &(state.state->strm); + + /* allocate input buffer (double size for gzprintf) */ + state.state->in = (unsigned char*)malloc(state.state->want << 1); + if (state.state->in == NULL) { + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* only need output buffer and deflate state if compressing */ + if (!state.state->direct) { + /* allocate output buffer */ + state.state->out = (unsigned char*)malloc(state.state->want); + if (state.state->out == NULL) { + free(state.state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + + /* allocate deflate memory, set up for gzip compression */ + strm->zalloc = Z_NULL; + strm->zfree = Z_NULL; + strm->opaque = Z_NULL; + ret = deflateInit2(strm, state.state->level, Z_DEFLATED, + MAX_WBITS + 16, DEF_MEM_LEVEL, state.state->strategy); + if (ret != Z_OK) { + free(state.state->out); + free(state.state->in); + gz_error(state, Z_MEM_ERROR, "out of memory"); + return -1; + } + strm->next_in = NULL; + } + + /* mark state as initialized */ + state.state->size = state.state->want; + + /* initialize write buffer if compressing */ + if (!state.state->direct) { + strm->avail_out = state.state->size; + strm->next_out = state.state->out; + state.state->x.next = strm->next_out; + } + return 0; +} + +/* Compress whatever is at avail_in and next_in and write to the output file. + Return -1 if there is an error writing to the output file or if gz_init() + fails to allocate memory, otherwise 0. flush is assumed to be a valid + deflate() flush value. If flush is Z_FINISH, then the deflate() state is + reset to start a new gzip stream. If gz->direct is true, then simply write + to the output file without compressing, and ignore flush. */ +local int gz_comp(state, flush) + gz_statep state; + int flush; +{ + int ret, writ; + unsigned have, put, max = ((unsigned)-1 >> 2) + 1; + z_streamp strm = &(state.state->strm); + + /* allocate memory if this is the first time through */ + if (state.state->size == 0 && gz_init(state) == -1) + return -1; + + /* write directly if requested */ + if (state.state->direct) { + while (strm->avail_in) { + put = strm->avail_in > max ? max : strm->avail_in; + writ = (int)write(state.state->fd, strm->next_in, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + strm->avail_in -= (unsigned)writ; + strm->next_in += writ; + } + return 0; + } + + /* run deflate() on provided input until it produces no more output */ + ret = Z_OK; + do { + /* write out current buffer contents if full, or if flushing, but if + doing Z_FINISH then don't write until we get to Z_STREAM_END */ + if (strm->avail_out == 0 || (flush != Z_NO_FLUSH && + (flush != Z_FINISH || ret == Z_STREAM_END))) { + while (strm->next_out > state.state->x.next) { + put = strm->next_out - state.state->x.next > (int)max ? max : + (unsigned)(strm->next_out - state.state->x.next); + writ = (int)write(state.state->fd, state.state->x.next, put); + if (writ < 0) { + gz_error(state, Z_ERRNO, zstrerror()); + return -1; + } + state.state->x.next += writ; + } + if (strm->avail_out == 0) { + strm->avail_out = state.state->size; + strm->next_out = state.state->out; + state.state->x.next = state.state->out; + } + } + + /* compress */ + have = strm->avail_out; + ret = deflate(strm, flush); + if (ret == Z_STREAM_ERROR) { + gz_error(state, Z_STREAM_ERROR, + "internal error: deflate stream corrupt"); + return -1; + } + have -= strm->avail_out; + } while (have); + + /* if that completed a deflate stream, allow another to start */ + if (flush == Z_FINISH) + deflateReset(strm); + + /* all done, no errors */ + return 0; +} + +/* Compress len zeros to output. Return -1 on a write error or memory + allocation failure by gz_comp(), or 0 on success. */ +local int gz_zero(state, len) + gz_statep state; + z_off64_t len; +{ + int first; + unsigned n; + z_streamp strm = &(state.state->strm); + + /* consume whatever's left in the input buffer */ + if (strm->avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return -1; + + /* compress len zeros (len guaranteed > 0) */ + first = 1; + while (len) { + n = GT_OFF(state.state->size) || (z_off64_t)state.state->size > len ? + (unsigned)len : state.state->size; + if (first) { + memset(state.state->in, 0, n); + first = 0; + } + strm->avail_in = n; + strm->next_in = state.state->in; + state.state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return -1; + len -= n; + } + return 0; +} + +/* Write len bytes from buf to file. Return the number of bytes written. If + the returned value is less than len, then there was an error. */ +local z_size_t gz_write(state, buf, len) + gz_statep state; + voidpc buf; + z_size_t len; +{ + z_size_t put = len; + + /* if len is zero, avoid unnecessary operations */ + if (len == 0) + return 0; + + /* allocate memory if this is the first time through */ + if (state.state->size == 0 && gz_init(state) == -1) + return 0; + + /* check for seek request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_zero(state, state.state->skip) == -1) + return 0; + } + + /* for small len, copy to input buffer, otherwise compress directly */ + if (len < state.state->size) { + /* copy to input buffer, compress when full */ + do { + z_size_t have, copy; + + if (state.state->strm.avail_in == 0) + state.state->strm.next_in = state.state->in; + have = (unsigned)((state.state->strm.next_in + state.state->strm.avail_in) - + state.state->in); + copy = state.state->size - have; + if (copy > len) + copy = len; + memcpy(state.state->in + have, buf, copy); + state.state->strm.avail_in += copy; + state.state->x.pos += copy; + buf = (const char *)buf + copy; + len -= copy; + if (len && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + } while (len); + } + else { + /* consume whatever's left in the input buffer */ + if (state.state->strm.avail_in && gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + + /* directly compress user buffer to file */ + state.state->strm.next_in = (z_const Bytef *)buf; + do { + z_size_t n = (unsigned)-1; + if (n > len) + n = len; + state.state->strm.avail_in = (z_uInt)n; + state.state->x.pos += n; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return 0; + len -= n; + } while (len); + } + + /* input was all buffered or compressed */ + return put; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzwrite(file, buf, len) + gzFile file; + voidpc buf; + unsigned len; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return 0; + state.file = file; + + /* check that we're writing and that there's no error */ + if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) + return 0; + + /* since an int is returned, make sure len fits in one, otherwise return + with an error (this avoids a flaw in the interface) */ + if ((int)len < 0) { + gz_error(state, Z_DATA_ERROR, "requested length does not fit in int"); + return 0; + } + + /* write len bytes from buf (the return value will fit in an int) */ + return (int)gz_write(state, buf, len); +} + +/* -- see zlib.h -- */ +z_size_t ZEXPORT gzfwrite(buf, size, nitems, file) + voidpc buf; + z_size_t size; + z_size_t nitems; + gzFile file; +{ + z_size_t len; + gz_statep state; + + /* get internal structure */ + assert(size != 0); + if (file == NULL) + return 0; + state.file = file; + + /* check that we're writing and that there's no error */ + if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) + return 0; + + /* compute bytes to read -- error on overflow */ + len = nitems * size; + if (size && (len / size != nitems)) { + gz_error(state, Z_STREAM_ERROR, "request does not fit in a size_t"); + return 0; + } + + /* write len bytes to buf, return the number of full items written */ + return len ? gz_write(state, buf, len) / size : 0; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzputc(file, c) + gzFile file; + int c; +{ + unsigned have; + unsigned char buf[1]; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return -1; + state.file = file; + strm = &(state.state->strm); + + /* check that we're writing and that there's no error */ + if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) + return -1; + + /* check for seek request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_zero(state, state.state->skip) == -1) + return -1; + } + + /* try writing to input buffer for speed (state.state->size == 0 if buffer not + initialized) */ + if (state.state->size) { + if (strm->avail_in == 0) + strm->next_in = state.state->in; + have = (unsigned)((strm->next_in + strm->avail_in) - state.state->in); + if (have < state.state->size) { + state.state->in[have] = (unsigned char)c; + strm->avail_in++; + state.state->x.pos++; + return c & 0xff; + } + } + + /* no room in buffer or not initialized, use gz_write() */ + buf[0] = (unsigned char)c; + if (gz_write(state, buf, 1) != 1) + return -1; + return c & 0xff; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzputs(file, str) + gzFile file; + const char *str; +{ + int ret; + z_size_t len; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return -1; + state.file = file; + + /* check that we're writing and that there's no error */ + if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) + return -1; + + /* write string */ + len = strlen(str); + ret = (int)gz_write(state, str, len); + return ret == 0 && len != 0 ? -1 : ret; +} + +#if defined(STDC) || defined(Z_HAVE_STDARG_H) +#include <stdarg.h> + +/* -- see zlib.h -- */ +int ZEXPORTVA gzvprintf(gzFile file, const char *format, va_list va) +{ + int len; + unsigned left; + char *next; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state.file = file; + strm = &(state.state->strm); + + /* check that we're writing and that there's no error */ + if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) + return Z_STREAM_ERROR; + + /* make sure we have some buffer space */ + if (state.state->size == 0 && gz_init(state) == -1) + return state.state->err; + + /* check for seek request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_zero(state, state.state->skip) == -1) + return state.state->err; + } + + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state.state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state.state->in; + next = (char *)(state.state->in + (strm->next_in - state.state->in) + strm->avail_in); + next[state.state->size - 1] = 0; +#ifdef NO_vsnprintf +# ifdef HAS_vsprintf_void + (void)vsprintf(next, format, va); + for (len = 0; len < state.state->size; len++) + if (next[len] == 0) break; +# else + len = vsprintf(next, format, va); +# endif +#else +# ifdef HAS_vsnprintf_void + (void)vsnprintf(next, state.state->size, format, va); + len = strlen(next); +# else + len = vsnprintf(next, state.state->size, format, va); +# endif +#endif + + /* check that printf() results fit in buffer */ + if (len == 0 || (unsigned)len >= state.state->size || next[state.state->size - 1] != 0) + return 0; + + /* update buffer and position, compress first half if past that */ + strm->avail_in += (unsigned)len; + state.state->x.pos += len; + if (strm->avail_in >= state.state->size) { + left = strm->avail_in - state.state->size; + strm->avail_in = state.state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state.state->err; + memcpy(state.state->in, state.state->in + state.state->size, left); + strm->next_in = state.state->in; + strm->avail_in = left; + } + return len; +} + +int ZEXPORTVA gzprintf(gzFile file, const char *format, ...) +{ + va_list va; + int ret; + + va_start(va, format); + ret = gzvprintf(file, format, va); + va_end(va); + return ret; +} + +#else /* !STDC && !Z_HAVE_STDARG_H */ + +/* -- see zlib.h -- */ +int ZEXPORTVA gzprintf (file, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20) + gzFile file; + const char *format; + int a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, + a11, a12, a13, a14, a15, a16, a17, a18, a19, a20; +{ + unsigned len, left; + char *next; + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state = (gz_statep)file; + strm = &(state.state->strm); + + /* check that can really pass pointer in ints */ + if (sizeof(int) != sizeof(void *)) + return Z_STREAM_ERROR; + + /* check that we're writing and that there's no error */ + if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) + return Z_STREAM_ERROR; + + /* make sure we have some buffer space */ + if (state.state->size == 0 && gz_init(state) == -1) + return state.state->error; + + /* check for seek request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_zero(state, state.state->skip) == -1) + return state.state->error; + } + + /* do the printf() into the input buffer, put length in len -- the input + buffer is double-sized just for this function, so there is guaranteed to + be state.state->size bytes available after the current contents */ + if (strm->avail_in == 0) + strm->next_in = state.state->in; + next = (char *)(strm->next_in + strm->avail_in); + next[state.state->size - 1] = 0; +#ifdef NO_snprintf +# ifdef HAS_sprintf_void + sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, a12, + a13, a14, a15, a16, a17, a18, a19, a20); + for (len = 0; len < size; len++) + if (next[len] == 0) + break; +# else + len = sprintf(next, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, a10, a11, + a12, a13, a14, a15, a16, a17, a18, a19, a20); +# endif +#else +# ifdef HAS_snprintf_void + snprintf(next, state.state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, a9, + a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); + len = strlen(next); +# else + len = snprintf(next, state.state->size, format, a1, a2, a3, a4, a5, a6, a7, a8, + a9, a10, a11, a12, a13, a14, a15, a16, a17, a18, a19, a20); +# endif +#endif + + /* check that printf() results fit in buffer */ + if (len == 0 || len >= state.state->size || next[state.state->size - 1] != 0) + return 0; + + /* update buffer and position, compress first half if past that */ + strm->avail_in += len; + state.state->x.pos += len; + if (strm->avail_in >= state.state->size) { + left = strm->avail_in - state.state->size; + strm->avail_in = state.state->size; + if (gz_comp(state, Z_NO_FLUSH) == -1) + return state.state->err; + memcpy(state.state->in, state.state->in + state.state->size, left); + strm->next_in = state.state->in; + strm->avail_in = left; + } + return (int)len; +} + +#endif + +/* -- see zlib.h -- */ +int ZEXPORT gzflush(file, flush) + gzFile file; + int flush; +{ + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state.file = file; + + /* check that we're writing and that there's no error */ + if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) + return Z_STREAM_ERROR; + + /* check flush parameter */ + if (flush < 0 || flush > Z_FINISH) + return Z_STREAM_ERROR; + + /* check for seek request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_zero(state, state.state->skip) == -1) + return state.state->err; + } + + /* compress remaining data with requested flush */ + (void)gz_comp(state, flush); + return state.state->err; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzsetparams(file, level, strategy) + gzFile file; + int level; + int strategy; +{ + gz_statep state; + z_streamp strm; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state.file = file; + strm = &(state.state->strm); + + /* check that we're writing and that there's no error */ + if (state.state->mode != GZ_WRITE || state.state->err != Z_OK) + return Z_STREAM_ERROR; + + /* if no change is requested, then do nothing */ + if (level == state.state->level && strategy == state.state->strategy) + return Z_OK; + + /* check for seek request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_zero(state, state.state->skip) == -1) + return state.state->err; + } + + /* change compression parameters for subsequent input */ + if (state.state->size) { + /* flush previous input with previous parameters before changing */ + if (strm->avail_in && gz_comp(state, Z_BLOCK) == -1) + return state.state->err; + deflateParams(strm, level, strategy); + } + state.state->level = level; + state.state->strategy = strategy; + return Z_OK; +} + +/* -- see zlib.h -- */ +int ZEXPORT gzclose_w(file) + gzFile file; +{ + int ret = Z_OK; + gz_statep state; + + /* get internal structure */ + if (file == NULL) + return Z_STREAM_ERROR; + state.file = file; + + /* check that we're writing */ + if (state.state->mode != GZ_WRITE) + return Z_STREAM_ERROR; + + /* check for seek request */ + if (state.state->seek) { + state.state->seek = 0; + if (gz_zero(state, state.state->skip) == -1) + ret = state.state->err; + } + + /* flush, free memory, and close file */ + if (gz_comp(state, Z_FINISH) == -1) + ret = state.state->err; + if (state.state->size) { + if (!state.state->direct) { + (void)deflateEnd(&(state.state->strm)); + free(state.state->out); + } + free(state.state->in); + } + gz_error(state, Z_OK, NULL); + free(state.state->path); + if (close(state.state->fd) == -1) + ret = Z_ERRNO; + free(state.state); + return ret; +} diff --git a/src/zstd/zlibWrapper/zstd_zlibwrapper.c b/src/zstd/zlibWrapper/zstd_zlibwrapper.c new file mode 100644 index 000000000..4d2c7be5b --- /dev/null +++ b/src/zstd/zlibWrapper/zstd_zlibwrapper.c @@ -0,0 +1,1109 @@ +/* + * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* === Tuning parameters === */ +#ifndef ZWRAP_USE_ZSTD + #define ZWRAP_USE_ZSTD 0 +#endif + + +/* === Dependencies === */ +#include <stdlib.h> +#include <stdio.h> /* vsprintf */ +#include <stdarg.h> /* va_list, for z_gzprintf */ +#define NO_DUMMY_DECL +#define ZLIB_CONST +#include <zlib.h> /* without #define Z_PREFIX */ +#include "zstd_zlibwrapper.h" +#define ZSTD_STATIC_LINKING_ONLY /* ZSTD_isFrame, ZSTD_MAGICNUMBER */ +#include "zstd.h" +#include "zstd_internal.h" /* ZSTD_malloc, ZSTD_free */ + + +/* === Constants === */ +#define Z_INFLATE_SYNC 8 +#define ZLIB_HEADERSIZE 4 +#define ZSTD_HEADERSIZE ZSTD_FRAMEHEADERSIZE_MIN(ZSTD_f_zstd1) +#define ZWRAP_DEFAULT_CLEVEL 3 /* Z_DEFAULT_COMPRESSION is translated to ZWRAP_DEFAULT_CLEVEL for zstd */ + + +/* === Debug === */ +#define LOG_WRAPPERC(...) /* fprintf(stderr, __VA_ARGS__) */ +#define LOG_WRAPPERD(...) /* fprintf(stderr, __VA_ARGS__) */ + +#define FINISH_WITH_GZ_ERR(msg) { (void)msg; return Z_STREAM_ERROR; } +#define FINISH_WITH_NULL_ERR(msg) { (void)msg; return NULL; } + + +/* === Wrapper === */ +static int g_ZWRAP_useZSTDcompression = ZWRAP_USE_ZSTD; /* 0 = don't use ZSTD */ + +void ZWRAP_useZSTDcompression(int turn_on) { g_ZWRAP_useZSTDcompression = turn_on; } + +int ZWRAP_isUsingZSTDcompression(void) { return g_ZWRAP_useZSTDcompression; } + + + +static ZWRAP_decompress_type g_ZWRAPdecompressionType = ZWRAP_AUTO; + +void ZWRAP_setDecompressionType(ZWRAP_decompress_type type) { g_ZWRAPdecompressionType = type; } + +ZWRAP_decompress_type ZWRAP_getDecompressionType(void) { return g_ZWRAPdecompressionType; } + + + +const char * zstdVersion(void) { return ZSTD_VERSION_STRING; } + +ZEXTERN const char * ZEXPORT z_zlibVersion OF((void)) { return zlibVersion(); } + + + +static void* ZWRAP_allocFunction(void* opaque, size_t size) +{ + z_streamp strm = (z_streamp) opaque; + void* address = strm->zalloc(strm->opaque, 1, (uInt)size); + /* LOG_WRAPPERC("ZWRAP alloc %p, %d \n", address, (int)size); */ + return address; +} + +static void ZWRAP_freeFunction(void* opaque, void* address) +{ + z_streamp strm = (z_streamp) opaque; + strm->zfree(strm->opaque, address); + /* if (address) LOG_WRAPPERC("ZWRAP free %p \n", address); */ +} + + + +/* === Compression === */ +typedef enum { ZWRAP_useInit, ZWRAP_useReset, ZWRAP_streamEnd } ZWRAP_state_t; + +typedef struct { + ZSTD_CStream* zbc; + int compressionLevel; + int streamEnd; /* a flag to signal the end of a stream */ + unsigned long long totalInBytes; /* we need it as strm->total_in can be reset by user */ + ZSTD_customMem customMem; + z_stream allocFunc; /* copy of zalloc, zfree, opaque */ + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; + ZWRAP_state_t comprState; + unsigned long long pledgedSrcSize; +} ZWRAP_CCtx; + +/* typedef ZWRAP_CCtx internal_state; */ + + + +static size_t ZWRAP_freeCCtx(ZWRAP_CCtx* zwc) +{ + if (zwc==NULL) return 0; /* support free on NULL */ + ZSTD_freeCStream(zwc->zbc); + ZSTD_free(zwc, zwc->customMem); + return 0; +} + + +static ZWRAP_CCtx* ZWRAP_createCCtx(z_streamp strm) +{ + ZWRAP_CCtx* zwc; + + if (strm->zalloc && strm->zfree) { + zwc = (ZWRAP_CCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_CCtx)); + if (zwc==NULL) return NULL; + memset(zwc, 0, sizeof(ZWRAP_CCtx)); + memcpy(&zwc->allocFunc, strm, sizeof(z_stream)); + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL }; + ZWRAP_customMem.opaque = &zwc->allocFunc; + zwc->customMem = ZWRAP_customMem; + } + } else { + zwc = (ZWRAP_CCtx*)calloc(1, sizeof(*zwc)); + if (zwc==NULL) return NULL; + } + + return zwc; +} + + +static int ZWRAP_initializeCStream(ZWRAP_CCtx* zwc, const void* dict, size_t dictSize, unsigned long long pledgedSrcSize) +{ + LOG_WRAPPERC("- ZWRAP_initializeCStream=%p\n", zwc); + if (zwc == NULL || zwc->zbc == NULL) return Z_STREAM_ERROR; + + if (!pledgedSrcSize) pledgedSrcSize = zwc->pledgedSrcSize; + { ZSTD_parameters const params = ZSTD_getParams(zwc->compressionLevel, pledgedSrcSize, dictSize); + size_t initErr; + LOG_WRAPPERC("pledgedSrcSize=%d windowLog=%d chainLog=%d hashLog=%d searchLog=%d minMatch=%d strategy=%d\n", + (int)pledgedSrcSize, params.cParams.windowLog, params.cParams.chainLog, params.cParams.hashLog, params.cParams.searchLog, params.cParams.minMatch, params.cParams.strategy); + initErr = ZSTD_initCStream_advanced(zwc->zbc, dict, dictSize, params, pledgedSrcSize); + if (ZSTD_isError(initErr)) return Z_STREAM_ERROR; + } + + return Z_OK; +} + + +static int ZWRAPC_finishWithError(ZWRAP_CCtx* zwc, z_streamp strm, int error) +{ + LOG_WRAPPERC("- ZWRAPC_finishWithError=%d\n", error); + if (zwc) ZWRAP_freeCCtx(zwc); + if (strm) strm->state = NULL; + return (error) ? error : Z_STREAM_ERROR; +} + + +static int ZWRAPC_finishWithErrorMsg(z_streamp strm, char* message) +{ + ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + strm->msg = message; + if (zwc == NULL) return Z_STREAM_ERROR; + + return ZWRAPC_finishWithError(zwc, strm, 0); +} + + +int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize) +{ + ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + if (zwc == NULL) return Z_STREAM_ERROR; + + zwc->pledgedSrcSize = pledgedSrcSize; + zwc->comprState = ZWRAP_useInit; + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_deflateInit_ OF((z_streamp strm, int level, + const char *version, int stream_size)) +{ + ZWRAP_CCtx* zwc; + + LOG_WRAPPERC("- deflateInit level=%d\n", level); + if (!g_ZWRAP_useZSTDcompression) { + return deflateInit_((strm), (level), version, stream_size); + } + + zwc = ZWRAP_createCCtx(strm); + if (zwc == NULL) return Z_MEM_ERROR; + + if (level == Z_DEFAULT_COMPRESSION) + level = ZWRAP_DEFAULT_CLEVEL; + + zwc->streamEnd = 0; + zwc->totalInBytes = 0; + zwc->compressionLevel = level; + strm->state = (struct internal_state*) zwc; /* use state which in not used by user */ + strm->total_in = 0; + strm->total_out = 0; + strm->adler = 0; + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_deflateInit2_ OF((z_streamp strm, int level, int method, + int windowBits, int memLevel, + int strategy, const char *version, + int stream_size)) +{ + if (!g_ZWRAP_useZSTDcompression) + return deflateInit2_(strm, level, method, windowBits, memLevel, strategy, version, stream_size); + + return z_deflateInit_ (strm, level, version, stream_size); +} + + +int ZWRAP_deflateReset_keepDict(z_streamp strm) +{ + LOG_WRAPPERC("- ZWRAP_deflateReset_keepDict\n"); + if (!g_ZWRAP_useZSTDcompression) + return deflateReset(strm); + + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + if (zwc) { + zwc->streamEnd = 0; + zwc->totalInBytes = 0; + } + } + + strm->total_in = 0; + strm->total_out = 0; + strm->adler = 0; + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_deflateReset OF((z_streamp strm)) +{ + LOG_WRAPPERC("- deflateReset\n"); + if (!g_ZWRAP_useZSTDcompression) + return deflateReset(strm); + + ZWRAP_deflateReset_keepDict(strm); + + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + if (zwc) zwc->comprState = ZWRAP_useInit; + } + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_deflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)) +{ + if (!g_ZWRAP_useZSTDcompression) { + LOG_WRAPPERC("- deflateSetDictionary\n"); + return deflateSetDictionary(strm, dictionary, dictLength); + } + + { ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + LOG_WRAPPERC("- deflateSetDictionary level=%d\n", (int)zwc->compressionLevel); + if (!zwc) return Z_STREAM_ERROR; + if (zwc->zbc == NULL) { + zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); + if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0); + } + { int res = ZWRAP_initializeCStream(zwc, dictionary, dictLength, ZSTD_CONTENTSIZE_UNKNOWN); + if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); } + zwc->comprState = ZWRAP_useReset; + } + + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_deflate OF((z_streamp strm, int flush)) +{ + ZWRAP_CCtx* zwc; + + if (!g_ZWRAP_useZSTDcompression) { + LOG_WRAPPERC("- deflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", + (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + return deflate(strm, flush); + } + + zwc = (ZWRAP_CCtx*) strm->state; + if (zwc == NULL) { LOG_WRAPPERC("zwc == NULL\n"); return Z_STREAM_ERROR; } + + if (zwc->zbc == NULL) { + zwc->zbc = ZSTD_createCStream_advanced(zwc->customMem); + if (zwc->zbc == NULL) return ZWRAPC_finishWithError(zwc, strm, 0); + { int const initErr = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : ZSTD_CONTENTSIZE_UNKNOWN); + if (initErr != Z_OK) return ZWRAPC_finishWithError(zwc, strm, initErr); } + if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset; + } else { + if (zwc->totalInBytes == 0) { + if (zwc->comprState == ZWRAP_useReset) { + size_t const resetErr = ZSTD_resetCStream(zwc->zbc, (flush == Z_FINISH) ? strm->avail_in : zwc->pledgedSrcSize); + if (ZSTD_isError(resetErr)) { + LOG_WRAPPERC("ERROR: ZSTD_resetCStream errorCode=%s\n", + ZSTD_getErrorName(resetErr)); + return ZWRAPC_finishWithError(zwc, strm, 0); + } + } else { + int const res = ZWRAP_initializeCStream(zwc, NULL, 0, (flush == Z_FINISH) ? strm->avail_in : ZSTD_CONTENTSIZE_UNKNOWN); + if (res != Z_OK) return ZWRAPC_finishWithError(zwc, strm, res); + if (flush != Z_FINISH) zwc->comprState = ZWRAP_useReset; + } + } /* (zwc->totalInBytes == 0) */ + } /* ! (zwc->zbc == NULL) */ + + LOG_WRAPPERC("- deflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + if (strm->avail_in > 0) { + zwc->inBuffer.src = strm->next_in; + zwc->inBuffer.size = strm->avail_in; + zwc->inBuffer.pos = 0; + zwc->outBuffer.dst = strm->next_out; + zwc->outBuffer.size = strm->avail_out; + zwc->outBuffer.pos = 0; + { size_t const cErr = ZSTD_compressStream(zwc->zbc, &zwc->outBuffer, &zwc->inBuffer); + LOG_WRAPPERC("deflate ZSTD_compressStream srcSize=%d dstCapacity=%d\n", (int)zwc->inBuffer.size, (int)zwc->outBuffer.size); + if (ZSTD_isError(cErr)) return ZWRAPC_finishWithError(zwc, strm, 0); + } + strm->next_out += zwc->outBuffer.pos; + strm->total_out += zwc->outBuffer.pos; + strm->avail_out -= zwc->outBuffer.pos; + strm->total_in += zwc->inBuffer.pos; + zwc->totalInBytes += zwc->inBuffer.pos; + strm->next_in += zwc->inBuffer.pos; + strm->avail_in -= zwc->inBuffer.pos; + } + + if (flush == Z_FULL_FLUSH +#if ZLIB_VERNUM >= 0x1240 + || flush == Z_TREES +#endif + || flush == Z_BLOCK) + return ZWRAPC_finishWithErrorMsg(strm, "Z_FULL_FLUSH, Z_BLOCK and Z_TREES are not supported!"); + + if (flush == Z_FINISH) { + size_t bytesLeft; + if (zwc->streamEnd) return Z_STREAM_END; + zwc->outBuffer.dst = strm->next_out; + zwc->outBuffer.size = strm->avail_out; + zwc->outBuffer.pos = 0; + bytesLeft = ZSTD_endStream(zwc->zbc, &zwc->outBuffer); + LOG_WRAPPERC("deflate ZSTD_endStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); + if (ZSTD_isError(bytesLeft)) return ZWRAPC_finishWithError(zwc, strm, 0); + strm->next_out += zwc->outBuffer.pos; + strm->total_out += zwc->outBuffer.pos; + strm->avail_out -= zwc->outBuffer.pos; + if (bytesLeft == 0) { + zwc->streamEnd = 1; + LOG_WRAPPERC("Z_STREAM_END2 strm->total_in=%d strm->avail_out=%d strm->total_out=%d\n", + (int)strm->total_in, (int)strm->avail_out, (int)strm->total_out); + return Z_STREAM_END; + } } + else + if (flush == Z_SYNC_FLUSH || flush == Z_PARTIAL_FLUSH) { + size_t bytesLeft; + zwc->outBuffer.dst = strm->next_out; + zwc->outBuffer.size = strm->avail_out; + zwc->outBuffer.pos = 0; + bytesLeft = ZSTD_flushStream(zwc->zbc, &zwc->outBuffer); + LOG_WRAPPERC("deflate ZSTD_flushStream dstCapacity=%d bytesLeft=%d\n", (int)strm->avail_out, (int)bytesLeft); + if (ZSTD_isError(bytesLeft)) return ZWRAPC_finishWithError(zwc, strm, 0); + strm->next_out += zwc->outBuffer.pos; + strm->total_out += zwc->outBuffer.pos; + strm->avail_out -= zwc->outBuffer.pos; + } + LOG_WRAPPERC("- deflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_deflateEnd OF((z_streamp strm)) +{ + if (!g_ZWRAP_useZSTDcompression) { + LOG_WRAPPERC("- deflateEnd\n"); + return deflateEnd(strm); + } + LOG_WRAPPERC("- deflateEnd total_in=%d total_out=%d\n", (int)(strm->total_in), (int)(strm->total_out)); + { size_t errorCode; + ZWRAP_CCtx* zwc = (ZWRAP_CCtx*) strm->state; + if (zwc == NULL) return Z_OK; /* structures are already freed */ + strm->state = NULL; + errorCode = ZWRAP_freeCCtx(zwc); + if (ZSTD_isError(errorCode)) return Z_STREAM_ERROR; + } + return Z_OK; +} + + +ZEXTERN uLong ZEXPORT z_deflateBound OF((z_streamp strm, + uLong sourceLen)) +{ + if (!g_ZWRAP_useZSTDcompression) + return deflateBound(strm, sourceLen); + + return ZSTD_compressBound(sourceLen); +} + + +ZEXTERN int ZEXPORT z_deflateParams OF((z_streamp strm, + int level, + int strategy)) +{ + if (!g_ZWRAP_useZSTDcompression) { + LOG_WRAPPERC("- deflateParams level=%d strategy=%d\n", level, strategy); + return deflateParams(strm, level, strategy); + } + + return Z_OK; +} + + + + + +/* === Decompression === */ + +typedef enum { ZWRAP_ZLIB_STREAM, ZWRAP_ZSTD_STREAM, ZWRAP_UNKNOWN_STREAM } ZWRAP_stream_type; + +typedef struct { + ZSTD_DStream* zbd; + char headerBuf[16]; /* must be >= ZSTD_frameHeaderSize_min */ + int errorCount; + unsigned long long totalInBytes; /* we need it as strm->total_in can be reset by user */ + ZWRAP_state_t decompState; + ZSTD_inBuffer inBuffer; + ZSTD_outBuffer outBuffer; + + /* zlib params */ + int stream_size; + char *version; + int windowBits; + ZSTD_customMem customMem; + z_stream allocFunc; /* just to copy zalloc, zfree, opaque */ +} ZWRAP_DCtx; + + +static void ZWRAP_initDCtx(ZWRAP_DCtx* zwd) +{ + zwd->errorCount = 0; + zwd->outBuffer.pos = 0; + zwd->outBuffer.size = 0; +} + +static ZWRAP_DCtx* ZWRAP_createDCtx(z_streamp strm) +{ + ZWRAP_DCtx* zwd; + MEM_STATIC_ASSERT(sizeof(zwd->headerBuf) >= ZSTD_HEADERSIZE); /* check static buffer size condition */ + + if (strm->zalloc && strm->zfree) { + zwd = (ZWRAP_DCtx*)strm->zalloc(strm->opaque, 1, sizeof(ZWRAP_DCtx)); + if (zwd==NULL) return NULL; + memset(zwd, 0, sizeof(ZWRAP_DCtx)); + zwd->allocFunc = *strm; /* just to copy zalloc, zfree & opaque */ + { ZSTD_customMem ZWRAP_customMem = { ZWRAP_allocFunction, ZWRAP_freeFunction, NULL }; + ZWRAP_customMem.opaque = &zwd->allocFunc; + zwd->customMem = ZWRAP_customMem; + } + } else { + zwd = (ZWRAP_DCtx*)calloc(1, sizeof(*zwd)); + if (zwd==NULL) return NULL; + } + + ZWRAP_initDCtx(zwd); + return zwd; +} + +static size_t ZWRAP_freeDCtx(ZWRAP_DCtx* zwd) +{ + if (zwd==NULL) return 0; /* support free on null */ + ZSTD_freeDStream(zwd->zbd); + ZSTD_free(zwd->version, zwd->customMem); + ZSTD_free(zwd, zwd->customMem); + return 0; +} + + +int ZWRAP_isUsingZSTDdecompression(z_streamp strm) +{ + if (strm == NULL) return 0; + return (strm->reserved == ZWRAP_ZSTD_STREAM); +} + + +static int ZWRAPD_finishWithError(ZWRAP_DCtx* zwd, z_streamp strm, int error) +{ + LOG_WRAPPERD("- ZWRAPD_finishWithError=%d\n", error); + ZWRAP_freeDCtx(zwd); + strm->state = NULL; + return (error) ? error : Z_STREAM_ERROR; +} + +static int ZWRAPD_finishWithErrorMsg(z_streamp strm, char* message) +{ + ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state; + strm->msg = message; + if (zwd == NULL) return Z_STREAM_ERROR; + + return ZWRAPD_finishWithError(zwd, strm, 0); +} + + +ZEXTERN int ZEXPORT z_inflateInit_ OF((z_streamp strm, + const char* version, int stream_size)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) { + strm->reserved = ZWRAP_ZLIB_STREAM; + return inflateInit(strm); + } + + { ZWRAP_DCtx* const zwd = ZWRAP_createDCtx(strm); + LOG_WRAPPERD("- inflateInit\n"); + if (zwd == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); + + zwd->version = (char*)ZSTD_malloc(strlen(version)+1, zwd->customMem); + if (zwd->version == NULL) return ZWRAPD_finishWithError(zwd, strm, 0); + strcpy(zwd->version, version); + + zwd->stream_size = stream_size; + zwd->totalInBytes = 0; + strm->state = (struct internal_state*) zwd; + strm->total_in = 0; + strm->total_out = 0; + strm->reserved = ZWRAP_UNKNOWN_STREAM; + strm->adler = 0; + } + + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_inflateInit2_ OF((z_streamp strm, int windowBits, + const char *version, int stream_size)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB) { + return inflateInit2_(strm, windowBits, version, stream_size); + } + + { int const ret = z_inflateInit_ (strm, version, stream_size); + LOG_WRAPPERD("- inflateInit2 windowBits=%d\n", windowBits); + if (ret == Z_OK) { + ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*)strm->state; + if (zwd == NULL) return Z_STREAM_ERROR; + zwd->windowBits = windowBits; + } + return ret; + } +} + +int ZWRAP_inflateReset_keepDict(z_streamp strm) +{ + LOG_WRAPPERD("- ZWRAP_inflateReset_keepDict\n"); + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateReset(strm); + + { ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state; + if (zwd == NULL) return Z_STREAM_ERROR; + ZWRAP_initDCtx(zwd); + zwd->decompState = ZWRAP_useReset; + zwd->totalInBytes = 0; + } + + strm->total_in = 0; + strm->total_out = 0; + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_inflateReset OF((z_streamp strm)) +{ + LOG_WRAPPERD("- inflateReset\n"); + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateReset(strm); + + { int const ret = ZWRAP_inflateReset_keepDict(strm); + if (ret != Z_OK) return ret; } + + { ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state; + if (zwd == NULL) return Z_STREAM_ERROR; + zwd->decompState = ZWRAP_useInit; } + + return Z_OK; +} + + +#if ZLIB_VERNUM >= 0x1240 +ZEXTERN int ZEXPORT z_inflateReset2 OF((z_streamp strm, + int windowBits)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateReset2(strm, windowBits); + + { int const ret = z_inflateReset (strm); + if (ret == Z_OK) { + ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*)strm->state; + if (zwd == NULL) return Z_STREAM_ERROR; + zwd->windowBits = windowBits; + } + return ret; + } +} +#endif + + +ZEXTERN int ZEXPORT z_inflateSetDictionary OF((z_streamp strm, + const Bytef *dictionary, + uInt dictLength)) +{ + LOG_WRAPPERD("- inflateSetDictionary\n"); + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateSetDictionary(strm, dictionary, dictLength); + + { ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state; + if (zwd == NULL || zwd->zbd == NULL) return Z_STREAM_ERROR; + { size_t const initErr = ZSTD_initDStream_usingDict(zwd->zbd, dictionary, dictLength); + if (ZSTD_isError(initErr)) return ZWRAPD_finishWithError(zwd, strm, 0); } + zwd->decompState = ZWRAP_useReset; + + if (zwd->totalInBytes == ZSTD_HEADERSIZE) { + zwd->inBuffer.src = zwd->headerBuf; + zwd->inBuffer.size = zwd->totalInBytes; + zwd->inBuffer.pos = 0; + zwd->outBuffer.dst = strm->next_out; + zwd->outBuffer.size = 0; + zwd->outBuffer.pos = 0; + { size_t const errorCode = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + LOG_WRAPPERD("inflateSetDictionary ZSTD_decompressStream errorCode=%d srcSize=%d dstCapacity=%d\n", + (int)errorCode, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); + if (zwd->inBuffer.pos < zwd->outBuffer.size || ZSTD_isError(errorCode)) { + LOG_WRAPPERD("ERROR: ZSTD_decompressStream %s\n", + ZSTD_getErrorName(errorCode)); + return ZWRAPD_finishWithError(zwd, strm, 0); + } } } } + + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_inflate OF((z_streamp strm, int flush)) +{ + ZWRAP_DCtx* zwd; + + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) { + int const result = inflate(strm, flush); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", + (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, result); + return result; + } + + if (strm->avail_in <= 0) return Z_OK; + + zwd = (ZWRAP_DCtx*) strm->state; + LOG_WRAPPERD("- inflate1 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d\n", + (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + + if (zwd == NULL) return Z_STREAM_ERROR; + if (zwd->decompState == ZWRAP_streamEnd) return Z_STREAM_END; + + if (zwd->totalInBytes < ZLIB_HEADERSIZE) { + if (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) { + if (MEM_readLE32(strm->next_in) != ZSTD_MAGICNUMBER) { + { int const initErr = (zwd->windowBits) ? + inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size) : + inflateInit_(strm, zwd->version, zwd->stream_size); + LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", initErr); + if (initErr != Z_OK) return ZWRAPD_finishWithError(zwd, strm, initErr); + } + + strm->reserved = ZWRAP_ZLIB_STREAM; + { size_t const freeErr = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(freeErr)) goto error; } + + { int const result = (flush == Z_INFLATE_SYNC) ? + inflateSync(strm) : + inflate(strm, flush); + LOG_WRAPPERD("- inflate3 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", + (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); + return result; + } } + } else { /* ! (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) */ + size_t const srcSize = MIN(strm->avail_in, ZLIB_HEADERSIZE - zwd->totalInBytes); + memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize); + strm->total_in += srcSize; + zwd->totalInBytes += srcSize; + strm->next_in += srcSize; + strm->avail_in -= srcSize; + if (zwd->totalInBytes < ZLIB_HEADERSIZE) return Z_OK; + + if (MEM_readLE32(zwd->headerBuf) != ZSTD_MAGICNUMBER) { + z_stream strm2; + strm2.next_in = strm->next_in; + strm2.avail_in = strm->avail_in; + strm2.next_out = strm->next_out; + strm2.avail_out = strm->avail_out; + + { int const initErr = (zwd->windowBits) ? + inflateInit2_(strm, zwd->windowBits, zwd->version, zwd->stream_size) : + inflateInit_(strm, zwd->version, zwd->stream_size); + LOG_WRAPPERD("ZLIB inflateInit errorCode=%d\n", initErr); + if (initErr != Z_OK) return ZWRAPD_finishWithError(zwd, strm, initErr); + } + + /* inflate header */ + strm->next_in = (unsigned char*)zwd->headerBuf; + strm->avail_in = ZLIB_HEADERSIZE; + strm->avail_out = 0; + { int const dErr = inflate(strm, Z_NO_FLUSH); + LOG_WRAPPERD("ZLIB inflate errorCode=%d strm->avail_in=%d\n", + dErr, (int)strm->avail_in); + if (dErr != Z_OK) + return ZWRAPD_finishWithError(zwd, strm, dErr); + } + if (strm->avail_in > 0) goto error; + + strm->next_in = strm2.next_in; + strm->avail_in = strm2.avail_in; + strm->next_out = strm2.next_out; + strm->avail_out = strm2.avail_out; + + strm->reserved = ZWRAP_ZLIB_STREAM; /* mark as zlib stream */ + { size_t const freeErr = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(freeErr)) goto error; } + + { int const result = (flush == Z_INFLATE_SYNC) ? + inflateSync(strm) : + inflate(strm, flush); + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", + (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, res); + return result; + } } } /* if ! (zwd->totalInBytes == 0 && strm->avail_in >= ZLIB_HEADERSIZE) */ + } /* (zwd->totalInBytes < ZLIB_HEADERSIZE) */ + + strm->reserved = ZWRAP_ZSTD_STREAM; /* mark as zstd steam */ + + if (flush == Z_INFLATE_SYNC) { strm->msg = "inflateSync is not supported!"; goto error; } + + if (!zwd->zbd) { + zwd->zbd = ZSTD_createDStream_advanced(zwd->customMem); + if (zwd->zbd == NULL) { LOG_WRAPPERD("ERROR: ZSTD_createDStream_advanced\n"); goto error; } + zwd->decompState = ZWRAP_useInit; + } + + if (zwd->totalInBytes < ZSTD_HEADERSIZE) { + if (zwd->totalInBytes == 0 && strm->avail_in >= ZSTD_HEADERSIZE) { + if (zwd->decompState == ZWRAP_useInit) { + size_t const initErr = ZSTD_initDStream(zwd->zbd); + if (ZSTD_isError(initErr)) { + LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", + ZSTD_getErrorName(initErr)); + goto error; + } + } else { + size_t const resetErr = ZSTD_resetDStream(zwd->zbd); + if (ZSTD_isError(resetErr)) goto error; + } + } else { + size_t const srcSize = MIN(strm->avail_in, ZSTD_HEADERSIZE - zwd->totalInBytes); + memcpy(zwd->headerBuf+zwd->totalInBytes, strm->next_in, srcSize); + strm->total_in += srcSize; + zwd->totalInBytes += srcSize; + strm->next_in += srcSize; + strm->avail_in -= srcSize; + if (zwd->totalInBytes < ZSTD_HEADERSIZE) return Z_OK; + + if (zwd->decompState == ZWRAP_useInit) { + size_t const initErr = ZSTD_initDStream(zwd->zbd); + if (ZSTD_isError(initErr)) { + LOG_WRAPPERD("ERROR: ZSTD_initDStream errorCode=%s\n", + ZSTD_getErrorName(initErr)); + goto error; + } + } else { + size_t const resetErr = ZSTD_resetDStream(zwd->zbd); + if (ZSTD_isError(resetErr)) goto error; + } + + zwd->inBuffer.src = zwd->headerBuf; + zwd->inBuffer.size = ZSTD_HEADERSIZE; + zwd->inBuffer.pos = 0; + zwd->outBuffer.dst = strm->next_out; + zwd->outBuffer.size = 0; + zwd->outBuffer.pos = 0; + { size_t const dErr = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + LOG_WRAPPERD("inflate ZSTD_decompressStream1 errorCode=%d srcSize=%d dstCapacity=%d\n", + (int)dErr, (int)zwd->inBuffer.size, (int)zwd->outBuffer.size); + if (ZSTD_isError(dErr)) { + LOG_WRAPPERD("ERROR: ZSTD_decompressStream1 %s\n", ZSTD_getErrorName(dErr)); + goto error; + } } + if (zwd->inBuffer.pos != zwd->inBuffer.size) goto error; /* not consumed */ + } + } /* (zwd->totalInBytes < ZSTD_HEADERSIZE) */ + + zwd->inBuffer.src = strm->next_in; + zwd->inBuffer.size = strm->avail_in; + zwd->inBuffer.pos = 0; + zwd->outBuffer.dst = strm->next_out; + zwd->outBuffer.size = strm->avail_out; + zwd->outBuffer.pos = 0; + { size_t const dErr = ZSTD_decompressStream(zwd->zbd, &zwd->outBuffer, &zwd->inBuffer); + LOG_WRAPPERD("inflate ZSTD_decompressStream2 errorCode=%d srcSize=%d dstCapacity=%d\n", + (int)dErr, (int)strm->avail_in, (int)strm->avail_out); + if (ZSTD_isError(dErr)) { + zwd->errorCount++; + LOG_WRAPPERD("ERROR: ZSTD_decompressStream2 %s zwd->errorCount=%d\n", + ZSTD_getErrorName(dErr), zwd->errorCount); + if (zwd->errorCount<=1) return Z_NEED_DICT; else goto error; + } + LOG_WRAPPERD("inflate inBuffer.pos=%d inBuffer.size=%d outBuffer.pos=%d outBuffer.size=%d o\n", + (int)zwd->inBuffer.pos, (int)zwd->inBuffer.size, (int)zwd->outBuffer.pos, (int)zwd->outBuffer.size); + strm->next_out += zwd->outBuffer.pos; + strm->total_out += zwd->outBuffer.pos; + strm->avail_out -= zwd->outBuffer.pos; + strm->total_in += zwd->inBuffer.pos; + zwd->totalInBytes += zwd->inBuffer.pos; + strm->next_in += zwd->inBuffer.pos; + strm->avail_in -= zwd->inBuffer.pos; + if (dErr == 0) { + LOG_WRAPPERD("inflate Z_STREAM_END1 avail_in=%d avail_out=%d total_in=%d total_out=%d\n", + (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out); + zwd->decompState = ZWRAP_streamEnd; + return Z_STREAM_END; + } + } /* dErr lifetime */ + + LOG_WRAPPERD("- inflate2 flush=%d avail_in=%d avail_out=%d total_in=%d total_out=%d res=%d\n", + (int)flush, (int)strm->avail_in, (int)strm->avail_out, (int)strm->total_in, (int)strm->total_out, Z_OK); + return Z_OK; + +error: + return ZWRAPD_finishWithError(zwd, strm, 0); +} + + +ZEXTERN int ZEXPORT z_inflateEnd OF((z_streamp strm)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateEnd(strm); + + LOG_WRAPPERD("- inflateEnd total_in=%d total_out=%d\n", + (int)(strm->total_in), (int)(strm->total_out)); + { ZWRAP_DCtx* const zwd = (ZWRAP_DCtx*) strm->state; + if (zwd == NULL) return Z_OK; /* structures are already freed */ + { size_t const freeErr = ZWRAP_freeDCtx(zwd); + if (ZSTD_isError(freeErr)) return Z_STREAM_ERROR; } + strm->state = NULL; + } + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_inflateSync OF((z_streamp strm)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) { + return inflateSync(strm); + } + + return z_inflate(strm, Z_INFLATE_SYNC); +} + + + +/* Advanced compression functions */ +ZEXTERN int ZEXPORT z_deflateCopy OF((z_streamp dest, + z_streamp source)) +{ + if (!g_ZWRAP_useZSTDcompression) + return deflateCopy(dest, source); + return ZWRAPC_finishWithErrorMsg(source, "deflateCopy is not supported!"); +} + + +ZEXTERN int ZEXPORT z_deflateTune OF((z_streamp strm, + int good_length, + int max_lazy, + int nice_length, + int max_chain)) +{ + if (!g_ZWRAP_useZSTDcompression) + return deflateTune(strm, good_length, max_lazy, nice_length, max_chain); + return ZWRAPC_finishWithErrorMsg(strm, "deflateTune is not supported!"); +} + + +#if ZLIB_VERNUM >= 0x1260 +ZEXTERN int ZEXPORT z_deflatePending OF((z_streamp strm, + unsigned *pending, + int *bits)) +{ + if (!g_ZWRAP_useZSTDcompression) + return deflatePending(strm, pending, bits); + return ZWRAPC_finishWithErrorMsg(strm, "deflatePending is not supported!"); +} +#endif + + +ZEXTERN int ZEXPORT z_deflatePrime OF((z_streamp strm, + int bits, + int value)) +{ + if (!g_ZWRAP_useZSTDcompression) + return deflatePrime(strm, bits, value); + return ZWRAPC_finishWithErrorMsg(strm, "deflatePrime is not supported!"); +} + + +ZEXTERN int ZEXPORT z_deflateSetHeader OF((z_streamp strm, + gz_headerp head)) +{ + if (!g_ZWRAP_useZSTDcompression) + return deflateSetHeader(strm, head); + return ZWRAPC_finishWithErrorMsg(strm, "deflateSetHeader is not supported!"); +} + + + + +/* Advanced decompression functions */ +#if ZLIB_VERNUM >= 0x1280 +ZEXTERN int ZEXPORT z_inflateGetDictionary OF((z_streamp strm, + Bytef *dictionary, + uInt *dictLength)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateGetDictionary(strm, dictionary, dictLength); + return ZWRAPD_finishWithErrorMsg(strm, "inflateGetDictionary is not supported!"); +} +#endif + + +ZEXTERN int ZEXPORT z_inflateCopy OF((z_streamp dest, + z_streamp source)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !source->reserved) + return inflateCopy(dest, source); + return ZWRAPD_finishWithErrorMsg(source, "inflateCopy is not supported!"); +} + + +#if ZLIB_VERNUM >= 0x1240 +ZEXTERN long ZEXPORT z_inflateMark OF((z_streamp strm)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateMark(strm); + return ZWRAPD_finishWithErrorMsg(strm, "inflateMark is not supported!"); +} +#endif + + +ZEXTERN int ZEXPORT z_inflatePrime OF((z_streamp strm, + int bits, + int value)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflatePrime(strm, bits, value); + return ZWRAPD_finishWithErrorMsg(strm, "inflatePrime is not supported!"); +} + + +ZEXTERN int ZEXPORT z_inflateGetHeader OF((z_streamp strm, + gz_headerp head)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateGetHeader(strm, head); + return ZWRAPD_finishWithErrorMsg(strm, "inflateGetHeader is not supported!"); +} + + +ZEXTERN int ZEXPORT z_inflateBackInit_ OF((z_streamp strm, int windowBits, + unsigned char FAR *window, + const char *version, + int stream_size)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateBackInit_(strm, windowBits, window, version, stream_size); + return ZWRAPD_finishWithErrorMsg(strm, "inflateBackInit is not supported!"); +} + + +ZEXTERN int ZEXPORT z_inflateBack OF((z_streamp strm, + in_func in, void FAR *in_desc, + out_func out, void FAR *out_desc)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateBack(strm, in, in_desc, out, out_desc); + return ZWRAPD_finishWithErrorMsg(strm, "inflateBack is not supported!"); +} + + +ZEXTERN int ZEXPORT z_inflateBackEnd OF((z_streamp strm)) +{ + if (g_ZWRAPdecompressionType == ZWRAP_FORCE_ZLIB || !strm->reserved) + return inflateBackEnd(strm); + return ZWRAPD_finishWithErrorMsg(strm, "inflateBackEnd is not supported!"); +} + + +ZEXTERN uLong ZEXPORT z_zlibCompileFlags OF((void)) { return zlibCompileFlags(); } + + + + /* === utility functions === */ +#ifndef Z_SOLO + +ZEXTERN int ZEXPORT z_compress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)) +{ + if (!g_ZWRAP_useZSTDcompression) + return compress(dest, destLen, source, sourceLen); + + { size_t dstCapacity = *destLen; + size_t const cSize = ZSTD_compress(dest, dstCapacity, + source, sourceLen, + ZWRAP_DEFAULT_CLEVEL); + LOG_WRAPPERD("z_compress sourceLen=%d dstCapacity=%d\n", + (int)sourceLen, (int)dstCapacity); + if (ZSTD_isError(cSize)) return Z_STREAM_ERROR; + *destLen = cSize; + } + return Z_OK; +} + + +ZEXTERN int ZEXPORT z_compress2 OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen, + int level)) +{ + if (!g_ZWRAP_useZSTDcompression) + return compress2(dest, destLen, source, sourceLen, level); + + { size_t dstCapacity = *destLen; + size_t const cSize = ZSTD_compress(dest, dstCapacity, source, sourceLen, level); + if (ZSTD_isError(cSize)) return Z_STREAM_ERROR; + *destLen = cSize; + } + return Z_OK; +} + + +ZEXTERN uLong ZEXPORT z_compressBound OF((uLong sourceLen)) +{ + if (!g_ZWRAP_useZSTDcompression) + return compressBound(sourceLen); + + return ZSTD_compressBound(sourceLen); +} + + +ZEXTERN int ZEXPORT z_uncompress OF((Bytef *dest, uLongf *destLen, + const Bytef *source, uLong sourceLen)) +{ + if (!ZSTD_isFrame(source, sourceLen)) + return uncompress(dest, destLen, source, sourceLen); + + { size_t dstCapacity = *destLen; + size_t const dSize = ZSTD_decompress(dest, dstCapacity, source, sourceLen); + if (ZSTD_isError(dSize)) return Z_STREAM_ERROR; + *destLen = dSize; + } + return Z_OK; +} + +#endif /* !Z_SOLO */ + + + /* checksum functions */ + +ZEXTERN uLong ZEXPORT z_adler32 OF((uLong adler, const Bytef *buf, uInt len)) +{ + return adler32(adler, buf, len); +} + +ZEXTERN uLong ZEXPORT z_crc32 OF((uLong crc, const Bytef *buf, uInt len)) +{ + return crc32(crc, buf, len); +} + + +#if ZLIB_VERNUM >= 0x12B0 +ZEXTERN uLong ZEXPORT z_adler32_z OF((uLong adler, const Bytef *buf, z_size_t len)) +{ + return adler32_z(adler, buf, len); +} + +ZEXTERN uLong ZEXPORT z_crc32_z OF((uLong crc, const Bytef *buf, z_size_t len)) +{ + return crc32_z(crc, buf, len); +} +#endif + + +#if ZLIB_VERNUM >= 0x1270 +ZEXTERN const z_crc_t FAR * ZEXPORT z_get_crc_table OF((void)) +{ + return get_crc_table(); +} +#endif diff --git a/src/zstd/zlibWrapper/zstd_zlibwrapper.h b/src/zstd/zlibWrapper/zstd_zlibwrapper.h new file mode 100644 index 000000000..e791043e1 --- /dev/null +++ b/src/zstd/zlibWrapper/zstd_zlibwrapper.h @@ -0,0 +1,88 @@ +/* + * Copyright (c) 2016-2020, Przemyslaw Skibinski, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#ifndef ZSTD_ZLIBWRAPPER_H +#define ZSTD_ZLIBWRAPPER_H + +#if defined (__cplusplus) +extern "C" { +#endif + + +#define ZLIB_CONST +#define Z_PREFIX +#define ZLIB_INTERNAL /* disables gz*64 functions but fixes zlib 1.2.4 with Z_PREFIX */ +#include <zlib.h> + +#if !defined(z_const) + #define z_const +#endif + + +/* returns a string with version of zstd library */ +const char * zstdVersion(void); + + +/*** COMPRESSION ***/ +/* ZWRAP_useZSTDcompression() enables/disables zstd compression during runtime. + By default zstd compression is disabled. To enable zstd compression please use one of the methods: + - compilation with the additional option -DZWRAP_USE_ZSTD=1 + - using '#define ZWRAP_USE_ZSTD 1' in source code before '#include "zstd_zlibwrapper.h"' + - calling ZWRAP_useZSTDcompression(1) + All above-mentioned methods will enable zstd compression for all threads. + Be aware that ZWRAP_useZSTDcompression() is not thread-safe and may lead to a race condition. */ +void ZWRAP_useZSTDcompression(int turn_on); + +/* checks if zstd compression is turned on */ +int ZWRAP_isUsingZSTDcompression(void); + +/* Changes a pledged source size for a given compression stream. + It will change ZSTD compression parameters what may improve compression speed and/or ratio. + The function should be called just after deflateInit() or deflateReset() and before deflate() or deflateSetDictionary(). + It's only helpful when data is compressed in blocks. + There will be no change in case of deflateInit() or deflateReset() immediately followed by deflate(strm, Z_FINISH) + as this case is automatically detected. */ +int ZWRAP_setPledgedSrcSize(z_streamp strm, unsigned long long pledgedSrcSize); + +/* Similar to deflateReset but preserves dictionary set using deflateSetDictionary. + It should improve compression speed because there will be less calls to deflateSetDictionary + When using zlib compression this method redirects to deflateReset. */ +int ZWRAP_deflateReset_keepDict(z_streamp strm); + + + +/*** DECOMPRESSION ***/ +typedef enum { ZWRAP_FORCE_ZLIB, ZWRAP_AUTO } ZWRAP_decompress_type; + +/* ZWRAP_setDecompressionType() enables/disables automatic recognition of zstd/zlib compressed data during runtime. + By default auto-detection of zstd and zlib streams in enabled (ZWRAP_AUTO). + Forcing zlib decompression with ZWRAP_setDecompressionType(ZWRAP_FORCE_ZLIB) slightly improves + decompression speed of zlib-encoded streams. + Be aware that ZWRAP_setDecompressionType() is not thread-safe and may lead to a race condition. */ +void ZWRAP_setDecompressionType(ZWRAP_decompress_type type); + +/* checks zstd decompression type */ +ZWRAP_decompress_type ZWRAP_getDecompressionType(void); + +/* Checks if zstd decompression is used for a given stream. + If will return 1 only when inflate() was called and zstd header was detected. */ +int ZWRAP_isUsingZSTDdecompression(z_streamp strm); + +/* Similar to inflateReset but preserves dictionary set using inflateSetDictionary. + inflate() will return Z_NEED_DICT only for the first time what will improve decompression speed. + For zlib streams this method redirects to inflateReset. */ +int ZWRAP_inflateReset_keepDict(z_streamp strm); + + +#if defined (__cplusplus) +} +#endif + +#endif /* ZSTD_ZLIBWRAPPER_H */ |