diff options
Diffstat (limited to 'src/zstd/examples')
| -rw-r--r-- | src/zstd/examples/.gitignore | 14 | ||||
| -rw-r--r-- | src/zstd/examples/Makefile | 83 | ||||
| -rw-r--r-- | src/zstd/examples/README.md | 40 | ||||
| -rw-r--r-- | src/zstd/examples/dictionary_compression.c | 157 | ||||
| -rw-r--r-- | src/zstd/examples/dictionary_decompression.c | 131 | ||||
| -rw-r--r-- | src/zstd/examples/multiple_streaming_compression.c | 165 | ||||
| -rw-r--r-- | src/zstd/examples/simple_compression.c | 135 | ||||
| -rw-r--r-- | src/zstd/examples/simple_decompression.c | 110 | ||||
| -rw-r--r-- | src/zstd/examples/streaming_compression.c | 131 | ||||
| -rw-r--r-- | src/zstd/examples/streaming_decompression.c | 116 | ||||
| -rw-r--r-- | src/zstd/examples/streaming_memory_usage.c | 149 |
11 files changed, 1231 insertions, 0 deletions
diff --git a/src/zstd/examples/.gitignore b/src/zstd/examples/.gitignore new file mode 100644 index 00000000..280feb36 --- /dev/null +++ b/src/zstd/examples/.gitignore @@ -0,0 +1,14 @@ +#build +simple_compression +simple_decompression +dictionary_compression +dictionary_decompression +streaming_compression +streaming_decompression +multiple_streaming_compression +streaming_memory_usage + +#test artefact +tmp* +test* +*.zst diff --git a/src/zstd/examples/Makefile b/src/zstd/examples/Makefile new file mode 100644 index 00000000..52470f59 --- /dev/null +++ b/src/zstd/examples/Makefile @@ -0,0 +1,83 @@ +# ################################################################ +# Copyright (c) 2016-present, Yann Collet, Facebook, Inc. +# All rights reserved. +# +# This source code is licensed under both the BSD-style license (found in the +# LICENSE file in the root directory of this source tree) and the GPLv2 (found +# in the COPYING file in the root directory of this source tree). +# ################################################################ + +# This Makefile presumes libzstd is installed, using `sudo make install` + +LIB = ../lib/libzstd.a + +.PHONY: default all clean test + +default: all + +all: simple_compression simple_decompression \ + dictionary_compression dictionary_decompression \ + streaming_compression streaming_decompression \ + multiple_streaming_compression streaming_memory_usage + +$(LIB) : + make -C ../lib libzstd.a + +simple_compression : simple_compression.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +simple_decompression : simple_decompression.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +dictionary_compression : dictionary_compression.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +dictionary_decompression : dictionary_decompression.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +streaming_compression : streaming_compression.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +multiple_streaming_compression : multiple_streaming_compression.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +streaming_decompression : streaming_decompression.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +streaming_memory_usage : streaming_memory_usage.c $(LIB) + $(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@ + +clean: + @rm -f core *.o tmp* result* *.zst \ + simple_compression simple_decompression \ + dictionary_compression dictionary_decompression \ + streaming_compression streaming_decompression \ + multiple_streaming_compression streaming_memory_usage + @echo Cleaning completed + +test: all + cp README.md tmp + cp Makefile tmp2 + @echo -- Simple compression tests + ./simple_compression tmp + ./simple_decompression tmp.zst + ./streaming_decompression tmp.zst > /dev/null + @echo -- Streaming memory usage + ./streaming_memory_usage + @echo -- Streaming compression tests + ./streaming_compression tmp + ./streaming_decompression tmp.zst > /dev/null + @echo -- Edge cases detection + ! ./streaming_decompression tmp # invalid input, must fail + ! ./simple_decompression tmp # invalid input, must fail + ! ./simple_decompression tmp.zst # unknown input size, must fail + touch tmpNull # create 0-size file + ./simple_compression tmpNull + ./simple_decompression tmpNull.zst # 0-size frame : must work + @echo -- Multiple streaming tests + ./multiple_streaming_compression *.c + @echo -- Dictionary compression tests + ./dictionary_compression tmp2 tmp README.md + ./dictionary_decompression tmp2.zst tmp.zst README.md + $(RM) tmp* *.zst + @echo tests completed diff --git a/src/zstd/examples/README.md b/src/zstd/examples/README.md new file mode 100644 index 00000000..eba50c99 --- /dev/null +++ b/src/zstd/examples/README.md @@ -0,0 +1,40 @@ +Zstandard library : usage examples +================================== + +- [Simple compression](simple_compression.c) : + Compress a single file. + Introduces usage of : `ZSTD_compress()` + +- [Simple decompression](simple_decompression.c) : + Decompress a single file. + Only compatible with simple compression. + Result remains in memory. + Introduces usage of : `ZSTD_decompress()` + +- [Streaming memory usage](streaming_memory_usage.c) : + Provides amount of memory used by streaming context + Introduces usage of : `ZSTD_sizeof_CStream()` + +- [Streaming compression](streaming_compression.c) : + Compress a single file. + Introduces usage of : `ZSTD_compressStream()` + +- [Multiple Streaming compression](multiple_streaming_compression.c) : + Compress multiple files in a single command line. + Introduces memory usage preservation technique, + reducing impact of malloc()/free() and memset() by re-using existing resources. + +- [Streaming decompression](streaming_decompression.c) : + Decompress a single file compressed by zstd. + Compatible with both simple and streaming compression. + Result is sent to stdout. + Introduces usage of : `ZSTD_decompressStream()` + +- [Dictionary compression](dictionary_compression.c) : + Compress multiple files using the same dictionary. + Introduces usage of : `ZSTD_createCDict()` and `ZSTD_compress_usingCDict()` + +- [Dictionary decompression](dictionary_decompression.c) : + Decompress multiple files using the same dictionary. + Result remains in memory. + Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()` diff --git a/src/zstd/examples/dictionary_compression.c b/src/zstd/examples/dictionary_compression.c new file mode 100644 index 00000000..97bf8cb5 --- /dev/null +++ b/src/zstd/examples/dictionary_compression.c @@ -0,0 +1,157 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#include <stdlib.h> // malloc, exit +#include <stdio.h> // printf +#include <string.h> // strerror +#include <errno.h> // errno +#include <sys/stat.h> // stat +#include <zstd.h> // presumes zstd library is installed + + +static off_t fsize_orDie(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + perror(filename); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(2); +} + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc"); + exit(3); +} + +static void* loadFile_orDie(const char* fileName, size_t* size) +{ + off_t const buffSize = fsize_orDie(fileName); + FILE* const inFile = fopen_orDie(fileName, "rb"); + void* const buffer = malloc_orDie(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); + exit(4); + } + fclose(inFile); + *size = buffSize; + return buffer; +} + +static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize) +{ + FILE* const oFile = fopen_orDie(fileName, "wb"); + size_t const wSize = fwrite(buff, 1, buffSize, oFile); + if (wSize != (size_t)buffSize) { + fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); + exit(5); + } + if (fclose(oFile)) { + perror(fileName); + exit(6); + } +} + +/* createDict() : + `dictFileName` is supposed to have been created using `zstd --train` */ +static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel) +{ + size_t dictSize; + printf("loading dictionary %s \n", dictFileName); + void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize); + ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel); + if (!cdict) { + fprintf(stderr, "ZSTD_createCDict error \n"); + exit(7); + } + free(dictBuffer); + return cdict; +} + + +static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict) +{ + size_t fSize; + void* const fBuff = loadFile_orDie(fname, &fSize); + size_t const cBuffSize = ZSTD_compressBound(fSize); + void* const cBuff = malloc_orDie(cBuffSize); + + ZSTD_CCtx* const cctx = ZSTD_createCCtx(); + if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); } + size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict); + if (ZSTD_isError(cSize)) { + fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); + exit(7); + } + + saveFile_orDie(oname, cBuff, cSize); + + /* success */ + printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); + + ZSTD_freeCCtx(cctx); /* never fails */ + free(fBuff); + free(cBuff); +} + + +static char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + int const cLevel = 3; + + if (argc<3) { + fprintf(stderr, "wrong arguments\n"); + fprintf(stderr, "usage:\n"); + fprintf(stderr, "%s [FILES] dictionary\n", exeName); + return 1; + } + + /* load dictionary only once */ + const char* const dictName = argv[argc-1]; + ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel); + + int u; + for (u=1; u<argc-1; u++) { + const char* inFilename = argv[u]; + char* const outFilename = createOutFilename_orDie(inFilename); + compress(inFilename, outFilename, dictPtr); + free(outFilename); + } + + ZSTD_freeCDict(dictPtr); + printf("All %u files compressed. \n", argc-2); + return 0; +} diff --git a/src/zstd/examples/dictionary_decompression.c b/src/zstd/examples/dictionary_decompression.c new file mode 100644 index 00000000..07e6e24c --- /dev/null +++ b/src/zstd/examples/dictionary_decompression.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +#include <stdlib.h> // malloc, exit +#include <stdio.h> // printf +#include <string.h> // strerror +#include <errno.h> // errno +#include <sys/stat.h> // stat +#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize +#include <zstd.h> // presumes zstd library is installed + + +static off_t fsize_orDie(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + perror(filename); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(2); +} + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc"); + exit(3); +} + +static void* loadFile_orDie(const char* fileName, size_t* size) +{ + off_t const buffSize = fsize_orDie(fileName); + FILE* const inFile = fopen_orDie(fileName, "rb"); + void* const buffer = malloc_orDie(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); + exit(4); + } + fclose(inFile); + *size = buffSize; + return buffer; +} + +/* createDict() : + `dictFileName` is supposed to have been created using `zstd --train` */ +static ZSTD_DDict* createDict_orDie(const char* dictFileName) +{ + size_t dictSize; + printf("loading dictionary %s \n", dictFileName); + void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize); + ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize); + if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); } + free(dictBuffer); + return ddict; +} + + +static void decompress(const char* fname, const ZSTD_DDict* ddict) +{ + size_t cSize; + void* const cBuff = loadFile_orDie(fname, &cSize); + unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); + if (rSize==ZSTD_CONTENTSIZE_ERROR) { + fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); + exit(5); + } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { + fprintf(stderr, "%s : original size unknown \n", fname); + exit(6); + } + + void* const rBuff = malloc_orDie((size_t)rSize); + + ZSTD_DCtx* const dctx = ZSTD_createDCtx(); + if (dctx==NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); } + size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict); + if (dSize != rSize) { + fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); + exit(7); + } + + /* success */ + printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); + + ZSTD_freeDCtx(dctx); + free(rBuff); + free(cBuff); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<3) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s [FILES] dictionary\n", exeName); + return 1; + } + + /* load dictionary only once */ + const char* const dictName = argv[argc-1]; + ZSTD_DDict* const dictPtr = createDict_orDie(dictName); + + int u; + for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr); + + ZSTD_freeDDict(dictPtr); + printf("All %u files correctly decoded (in memory) \n", argc-2); + return 0; +} diff --git a/src/zstd/examples/multiple_streaming_compression.c b/src/zstd/examples/multiple_streaming_compression.c new file mode 100644 index 00000000..e395aefb --- /dev/null +++ b/src/zstd/examples/multiple_streaming_compression.c @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/* The objective of this example is to show of to compress multiple successive files +* while preserving memory management. +* All structures and buffers will be created only once, +* and shared across all compression operations */ + +#include <stdlib.h> // malloc, exit +#include <stdio.h> // fprintf, perror, feof +#include <string.h> // strerror +#include <errno.h> // errno +#define ZSTD_STATIC_LINKING_ONLY // streaming API defined as "experimental" for the time being +#include <zstd.h> // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc:"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +typedef struct { + void* buffIn; + void* buffOut; + size_t buffInSize; + size_t buffOutSize; + ZSTD_CStream* cstream; +} resources ; + +static resources createResources_orDie() +{ + resources ress; + ress.buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */ + ress.buffOutSize= ZSTD_CStreamOutSize(); /* can always flush a full block */ + ress.buffIn = malloc_orDie(ress.buffInSize); + ress.buffOut= malloc_orDie(ress.buffOutSize); + ress.cstream = ZSTD_createCStream(); + if (ress.cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } + return ress; +} + +static void freeResources(resources ress) +{ + ZSTD_freeCStream(ress.cstream); + free(ress.buffIn); + free(ress.buffOut); +} + + +static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + FILE* const fout = fopen_orDie(outName, "wb"); + + size_t const initResult = ZSTD_initCStream(ress.cstream, cLevel); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } + + size_t read, toRead = ress.buffInSize; + while( (read = fread_orDie(ress.buffIn, toRead, fin)) ) { + ZSTD_inBuffer input = { ress.buffIn, read, 0 }; + while (input.pos < input.size) { + ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 }; + toRead = ZSTD_compressStream(ress.cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ + if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } + if (toRead > ress.buffInSize) toRead = ress.buffInSize; /* Safely handle when `buffInSize` is manually changed to a smaller value */ + fwrite_orDie(ress.buffOut, output.pos, fout); + } + } + + ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 }; + size_t const remainingToFlush = ZSTD_endStream(ress.cstream, &output); /* close frame */ + if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); } + fwrite_orDie(ress.buffOut, output.pos, fout); + + fclose_orDie(fout); + fclose_orDie(fin); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc<2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE(s)\n", exeName); + return 1; + } + + resources const ress = createResources_orDie(); + void* ofnBuffer = NULL; + size_t ofnbSize = 0; + + int argNb; + for (argNb = 1; argNb < argc; argNb++) { + const char* const ifn = argv[argNb]; + size_t const ifnSize = strlen(ifn); + size_t const ofnSize = ifnSize + 5; + if (ofnbSize <= ofnSize) { + ofnbSize = ofnSize + 16; + free(ofnBuffer); + ofnBuffer = malloc_orDie(ofnbSize); + } + memset(ofnBuffer, 0, ofnSize); + strcat(ofnBuffer, ifn); + strcat(ofnBuffer, ".zst"); + compressFile_orDie(ress, ifn, ofnBuffer, 7); + } + + freeResources(ress); + /* success */ + printf("compressed %i files \n", argc-1); + + return 0; +} diff --git a/src/zstd/examples/simple_compression.c b/src/zstd/examples/simple_compression.c new file mode 100644 index 00000000..9ade424a --- /dev/null +++ b/src/zstd/examples/simple_compression.c @@ -0,0 +1,135 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + + +#include <stdlib.h> // malloc, free, exit +#include <stdio.h> // fprintf, perror, fopen, etc. +#include <string.h> // strlen, strcat, memset, strerror +#include <errno.h> // errno +#include <sys/stat.h> // stat +#include <zstd.h> // presumes zstd library is installed + + +static off_t fsize_orDie(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + perror(filename); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(2); +} + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror(NULL); + exit(3); +} + +static void* loadFile_orDie(const char* fileName, size_t* size) +{ + off_t const fileSize = fsize_orDie(fileName); + size_t const buffSize = (size_t)fileSize; + if ((off_t)buffSize < fileSize) { /* narrowcast overflow */ + fprintf(stderr, "%s : filesize too large \n", fileName); + exit(4); + } + FILE* const inFile = fopen_orDie(fileName, "rb"); + void* const buffer = malloc_orDie(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); + exit(5); + } + fclose(inFile); /* can't fail, read only */ + *size = buffSize; + return buffer; +} + + +static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize) +{ + FILE* const oFile = fopen_orDie(fileName, "wb"); + size_t const wSize = fwrite(buff, 1, buffSize, oFile); + if (wSize != (size_t)buffSize) { + fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno)); + exit(6); + } + if (fclose(oFile)) { + perror(fileName); + exit(7); + } +} + + +static void compress_orDie(const char* fname, const char* oname) +{ + size_t fSize; + void* const fBuff = loadFile_orDie(fname, &fSize); + size_t const cBuffSize = ZSTD_compressBound(fSize); + void* const cBuff = malloc_orDie(cBuffSize); + + size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1); + if (ZSTD_isError(cSize)) { + fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize)); + exit(8); + } + + saveFile_orDie(oname, cBuff, cSize); + + /* success */ + printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname); + + free(fBuff); + free(cBuff); +} + + +static char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* const outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + const char* const inFilename = argv[1]; + + char* const outFilename = createOutFilename_orDie(inFilename); + compress_orDie(inFilename, outFilename); + free(outFilename); + return 0; +} diff --git a/src/zstd/examples/simple_decompression.c b/src/zstd/examples/simple_decompression.c new file mode 100644 index 00000000..c1818a95 --- /dev/null +++ b/src/zstd/examples/simple_decompression.c @@ -0,0 +1,110 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + +#include <stdlib.h> // malloc, exit +#include <stdio.h> // printf +#include <string.h> // strerror +#include <errno.h> // errno +#include <sys/stat.h> // stat +#define ZSTD_STATIC_LINKING_ONLY // ZSTD_findDecompressedSize +#include <zstd.h> // presumes zstd library is installed + + +static off_t fsize_orDie(const char *filename) +{ + struct stat st; + if (stat(filename, &st) == 0) return st.st_size; + /* error */ + fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno)); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno)); + exit(2); +} + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size + !size); /* avoid allocating size of 0 : may return NULL (implementation dependent) */ + if (buff) return buff; + /* error */ + fprintf(stderr, "malloc: %s \n", strerror(errno)); + exit(3); +} + +static void* loadFile_orDie(const char* fileName, size_t* size) +{ + off_t const buffSize = fsize_orDie(fileName); + FILE* const inFile = fopen_orDie(fileName, "rb"); + void* const buffer = malloc_orDie(buffSize); + size_t const readSize = fread(buffer, 1, buffSize, inFile); + if (readSize != (size_t)buffSize) { + fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno)); + exit(4); + } + fclose(inFile); /* can't fail (read only) */ + *size = buffSize; + return buffer; +} + + +static void decompress(const char* fname) +{ + size_t cSize; + void* const cBuff = loadFile_orDie(fname, &cSize); + unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize); + if (rSize==ZSTD_CONTENTSIZE_ERROR) { + fprintf(stderr, "%s : it was not compressed by zstd.\n", fname); + exit(5); + } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) { + fprintf(stderr, + "%s : original size unknown. Use streaming decompression instead.\n", fname); + exit(6); + } + + void* const rBuff = malloc_orDie((size_t)rSize); + + size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize); + + if (dSize != rSize) { + fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize)); + exit(7); + } + + /* success */ + printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize); + + free(rBuff); + free(cBuff); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + decompress(argv[1]); + + printf("%s correctly decoded (in memory). \n", argv[1]); + + return 0; +} diff --git a/src/zstd/examples/streaming_compression.c b/src/zstd/examples/streaming_compression.c new file mode 100644 index 00000000..f76364d8 --- /dev/null +++ b/src/zstd/examples/streaming_compression.c @@ -0,0 +1,131 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#include <stdlib.h> // malloc, free, exit +#include <stdio.h> // fprintf, perror, feof, fopen, etc. +#include <string.h> // strlen, memset, strcat +#include <zstd.h> // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc:"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +static void compressFile_orDie(const char* fname, const char* outName, int cLevel) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + FILE* const fout = fopen_orDie(outName, "wb"); + size_t const buffInSize = ZSTD_CStreamInSize(); /* can always read one full block */ + void* const buffIn = malloc_orDie(buffInSize); + size_t const buffOutSize = ZSTD_CStreamOutSize(); /* can always flush a full block */ + void* const buffOut = malloc_orDie(buffOutSize); + + ZSTD_CStream* const cstream = ZSTD_createCStream(); + if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); } + size_t const initResult = ZSTD_initCStream(cstream, cLevel); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } + + size_t read, toRead = buffInSize; + while( (read = fread_orDie(buffIn, toRead, fin)) ) { + ZSTD_inBuffer input = { buffIn, read, 0 }; + while (input.pos < input.size) { + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_compressStream(cstream, &output , &input); /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */ + if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } + if (toRead > buffInSize) toRead = buffInSize; /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/ + fwrite_orDie(buffOut, output.pos, fout); + } + } + + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + size_t const remainingToFlush = ZSTD_endStream(cstream, &output); /* close frame */ + if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); } + fwrite_orDie(buffOut, output.pos, fout); + + ZSTD_freeCStream(cstream); + fclose_orDie(fout); + fclose_orDie(fin); + free(buffIn); + free(buffOut); +} + + +static const char* createOutFilename_orDie(const char* filename) +{ + size_t const inL = strlen(filename); + size_t const outL = inL + 5; + void* outSpace = malloc_orDie(outL); + memset(outSpace, 0, outL); + strcat(outSpace, filename); + strcat(outSpace, ".zst"); + return (const char*)outSpace; +} + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc!=2) { + printf("wrong arguments\n"); + printf("usage:\n"); + printf("%s FILE\n", exeName); + return 1; + } + + const char* const inFilename = argv[1]; + + const char* const outFilename = createOutFilename_orDie(inFilename); + compressFile_orDie(inFilename, outFilename, 1); + + return 0; +} diff --git a/src/zstd/examples/streaming_decompression.c b/src/zstd/examples/streaming_decompression.c new file mode 100644 index 00000000..504a5e31 --- /dev/null +++ b/src/zstd/examples/streaming_decompression.c @@ -0,0 +1,116 @@ +/* + * Copyright (c) 2016-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +#include <stdlib.h> // malloc, exit +#include <stdio.h> // fprintf, perror, feof +#include <string.h> // strerror +#include <errno.h> // errno +#include <zstd.h> // presumes zstd library is installed + + +static void* malloc_orDie(size_t size) +{ + void* const buff = malloc(size); + if (buff) return buff; + /* error */ + perror("malloc:"); + exit(1); +} + +static FILE* fopen_orDie(const char *filename, const char *instruction) +{ + FILE* const inFile = fopen(filename, instruction); + if (inFile) return inFile; + /* error */ + perror(filename); + exit(3); +} + +static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file) +{ + size_t const readSize = fread(buffer, 1, sizeToRead, file); + if (readSize == sizeToRead) return readSize; /* good */ + if (feof(file)) return readSize; /* good, reached end of file */ + /* error */ + perror("fread"); + exit(4); +} + +static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file) +{ + size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file); + if (writtenSize == sizeToWrite) return sizeToWrite; /* good */ + /* error */ + perror("fwrite"); + exit(5); +} + +static size_t fclose_orDie(FILE* file) +{ + if (!fclose(file)) return 0; + /* error */ + perror("fclose"); + exit(6); +} + + +static void decompressFile_orDie(const char* fname) +{ + FILE* const fin = fopen_orDie(fname, "rb"); + size_t const buffInSize = ZSTD_DStreamInSize(); + void* const buffIn = malloc_orDie(buffInSize); + FILE* const fout = stdout; + size_t const buffOutSize = ZSTD_DStreamOutSize(); /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */ + void* const buffOut = malloc_orDie(buffOutSize); + + ZSTD_DStream* const dstream = ZSTD_createDStream(); + if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); } + + /* In more complex scenarios, a file may consist of multiple appended frames (ex : pzstd). + * The following example decompresses only the first frame. + * It is compatible with other provided streaming examples */ + size_t const initResult = ZSTD_initDStream(dstream); + if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); } + size_t read, toRead = initResult; + while ( (read = fread_orDie(buffIn, toRead, fin)) ) { + ZSTD_inBuffer input = { buffIn, read, 0 }; + while (input.pos < input.size) { + ZSTD_outBuffer output = { buffOut, buffOutSize, 0 }; + toRead = ZSTD_decompressStream(dstream, &output , &input); /* toRead : size of next compressed block */ + if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_decompressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); } + fwrite_orDie(buffOut, output.pos, fout); + } + } + + ZSTD_freeDStream(dstream); + fclose_orDie(fin); + fclose_orDie(fout); + free(buffIn); + free(buffOut); +} + + +int main(int argc, const char** argv) +{ + const char* const exeName = argv[0]; + + if (argc!=2) { + fprintf(stderr, "wrong arguments\n"); + fprintf(stderr, "usage:\n"); + fprintf(stderr, "%s FILE\n", exeName); + return 1; + } + + const char* const inFilename = argv[1]; + + decompressFile_orDie(inFilename); + return 0; +} diff --git a/src/zstd/examples/streaming_memory_usage.c b/src/zstd/examples/streaming_memory_usage.c new file mode 100644 index 00000000..b056c2a5 --- /dev/null +++ b/src/zstd/examples/streaming_memory_usage.c @@ -0,0 +1,149 @@ +/* + * Copyright (c) 2017-present, Yann Collet, Facebook, Inc. + * All rights reserved. + * + * This source code is licensed under both the BSD-style license (found in the + * LICENSE file in the root directory of this source tree) and the GPLv2 (found + * in the COPYING file in the root directory of this source tree). + * You may select, at your option, one of the above-listed licenses. + */ + + +/*=== Tuning parameter ===*/ +#ifndef MAX_TESTED_LEVEL +#define MAX_TESTED_LEVEL 12 +#endif + + +/*=== Dependencies ===*/ +#include <stdio.h> /* printf */ +#define ZSTD_STATIC_LINKING_ONLY +#include "zstd.h" + + +/*=== functions ===*/ + +/*! readU32FromChar() : + @return : unsigned integer value read from input in `char` format + allows and interprets K, KB, KiB, M, MB and MiB suffix. + Will also modify `*stringPtr`, advancing it to position where it stopped reading. + Note : function result can overflow if digit string > MAX_UINT */ +static unsigned readU32FromChar(const char** stringPtr) +{ + unsigned result = 0; + while ((**stringPtr >='0') && (**stringPtr <='9')) + result *= 10, result += **stringPtr - '0', (*stringPtr)++ ; + if ((**stringPtr=='K') || (**stringPtr=='M')) { + result <<= 10; + if (**stringPtr=='M') result <<= 10; + (*stringPtr)++ ; + if (**stringPtr=='i') (*stringPtr)++; + if (**stringPtr=='B') (*stringPtr)++; + } + return result; +} + + +int main(int argc, char const *argv[]) { + + printf("\n Zstandard (v%u) memory usage for streaming contexts : \n\n", ZSTD_versionNumber()); + + unsigned wLog = 0; + if (argc > 1) { + const char* valStr = argv[1]; + wLog = readU32FromChar(&valStr); + } + + int compressionLevel; + for (compressionLevel = 1; compressionLevel <= MAX_TESTED_LEVEL; compressionLevel++) { +#define INPUT_SIZE 5 +#define COMPRESSED_SIZE 128 + char const dataToCompress[INPUT_SIZE] = "abcde"; + char compressedData[COMPRESSED_SIZE]; + char decompressedData[INPUT_SIZE]; + ZSTD_CStream* const cstream = ZSTD_createCStream(); + if (cstream==NULL) { + printf("Level %i : ZSTD_CStream Memory allocation failure \n", compressionLevel); + return 1; + } + + /* forces compressor to use maximum memory size for given compression level, + * by not providing any information on input size */ + ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0); + if (wLog) { /* special mode : specific wLog */ + printf("Using custom compression parameter : level 1 + wLog=%u \n", wLog); + params = ZSTD_getParams(1, 1 << wLog, 0); + size_t const error = ZSTD_initCStream_advanced(cstream, NULL, 0, params, 0); + if (ZSTD_isError(error)) { + printf("ZSTD_initCStream_advanced error : %s \n", ZSTD_getErrorName(error)); + return 1; + } + } else { + size_t const error = ZSTD_initCStream(cstream, compressionLevel); + if (ZSTD_isError(error)) { + printf("ZSTD_initCStream error : %s \n", ZSTD_getErrorName(error)); + return 1; + } + } + + size_t compressedSize; + { ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 }; + ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 }; + size_t const cError = ZSTD_compressStream(cstream, &outBuff, &inBuff); + if (ZSTD_isError(cError)) { + printf("ZSTD_compressStream error : %s \n", ZSTD_getErrorName(cError)); + return 1; + } + size_t const fError = ZSTD_endStream(cstream, &outBuff); + if (ZSTD_isError(fError)) { + printf("ZSTD_endStream error : %s \n", ZSTD_getErrorName(fError)); + return 1; + } + compressedSize = outBuff.pos; + } + + ZSTD_DStream* dstream = ZSTD_createDStream(); + if (dstream==NULL) { + printf("Level %i : ZSTD_DStream Memory allocation failure \n", compressionLevel); + return 1; + } + { size_t const error = ZSTD_initDStream(dstream); + if (ZSTD_isError(error)) { + printf("ZSTD_initDStream error : %s \n", ZSTD_getErrorName(error)); + return 1; + } + } + /* forces decompressor to use maximum memory size, as decompressed size is not known */ + { ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 }; + ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 }; + size_t const dResult = ZSTD_decompressStream(dstream, &outBuff, &inBuff); + if (ZSTD_isError(dResult)) { + printf("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(dResult)); + return 1; + } + if (dResult != 0) { + printf("ZSTD_decompressStream error : unfinished decompression \n"); + return 1; + } + if (outBuff.pos != sizeof(dataToCompress)) { + printf("ZSTD_decompressStream error : incorrect decompression \n"); + return 1; + } + } + + size_t const cstreamSize = ZSTD_sizeof_CStream(cstream); + size_t const cstreamEstimatedSize = wLog ? + ZSTD_estimateCStreamSize_usingCParams(params.cParams) : + ZSTD_estimateCStreamSize(compressionLevel); + size_t const dstreamSize = ZSTD_sizeof_DStream(dstream); + + printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB \n", + compressionLevel, + (unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10), (unsigned)(dstreamSize>>10)); + + ZSTD_freeDStream(dstream); + ZSTD_freeCStream(cstream); + if (wLog) break; /* single test */ + } + return 0; +} |