Adding upstream version 14.2.21.upstream/14.2.21upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

11 files changed, 1231 insertions, 0 deletions

diff --git a/src/zstd/examples/.gitignore b/src/zstd/examples/.gitignore
new file mode 100644
index 00000000..280feb36
--- /dev/null
+++ b/src/zstd/examples/.gitignore
@@ -0,0 +1,14 @@
+#build
+simple_compression
+simple_decompression
+dictionary_compression
+dictionary_decompression
+streaming_compression
+streaming_decompression
+multiple_streaming_compression
+streaming_memory_usage
+
+#test artefact
+tmp*
+test*
+*.zst
diff --git a/src/zstd/examples/Makefile b/src/zstd/examples/Makefile
new file mode 100644
index 00000000..52470f59
--- /dev/null
+++ b/src/zstd/examples/Makefile
@@ -0,0 +1,83 @@
+# ################################################################
+# Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+# All rights reserved.
+#
+# This source code is licensed under both the BSD-style license (found in the
+# LICENSE file in the root directory of this source tree) and the GPLv2 (found
+# in the COPYING file in the root directory of this source tree).
+# ################################################################
+
+# This Makefile presumes libzstd is installed, using `sudo make install`
+
+LIB = ../lib/libzstd.a
+
+.PHONY: default all clean test
+
+default: all
+
+all: simple_compression simple_decompression \
+	dictionary_compression dictionary_decompression \
+	streaming_compression streaming_decompression \
+	multiple_streaming_compression streaming_memory_usage
+
+$(LIB) :
+	make -C ../lib libzstd.a
+
+simple_compression : simple_compression.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+simple_decompression : simple_decompression.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+dictionary_compression : dictionary_compression.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+dictionary_decompression : dictionary_decompression.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+streaming_compression : streaming_compression.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+multiple_streaming_compression : multiple_streaming_compression.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+streaming_decompression : streaming_decompression.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+streaming_memory_usage : streaming_memory_usage.c $(LIB)
+	$(CC) $(CPPFLAGS) $(CFLAGS) $^ $(LDFLAGS) -o $@
+
+clean:
+	@rm -f core *.o tmp* result* *.zst \
+        simple_compression simple_decompression \
+        dictionary_compression dictionary_decompression \
+        streaming_compression streaming_decompression \
+        multiple_streaming_compression streaming_memory_usage
+	@echo Cleaning completed
+
+test: all
+	cp README.md tmp
+	cp Makefile tmp2
+	@echo -- Simple compression tests
+	./simple_compression tmp
+	./simple_decompression tmp.zst
+	./streaming_decompression tmp.zst > /dev/null
+	@echo -- Streaming memory usage
+	./streaming_memory_usage
+	@echo -- Streaming compression tests
+	./streaming_compression tmp
+	./streaming_decompression tmp.zst > /dev/null
+	@echo -- Edge cases detection
+	! ./streaming_decompression tmp    # invalid input, must fail
+	! ./simple_decompression tmp       # invalid input, must fail
+	! ./simple_decompression tmp.zst   # unknown input size, must fail
+	touch tmpNull                      # create 0-size file
+	./simple_compression tmpNull
+	./simple_decompression tmpNull.zst # 0-size frame : must work
+	@echo -- Multiple streaming tests
+	./multiple_streaming_compression *.c
+	@echo -- Dictionary compression tests
+	./dictionary_compression tmp2 tmp README.md
+	./dictionary_decompression tmp2.zst tmp.zst README.md
+	$(RM) tmp* *.zst
+	@echo tests completed
diff --git a/src/zstd/examples/README.md b/src/zstd/examples/README.md
new file mode 100644
index 00000000..eba50c99
--- /dev/null
+++ b/src/zstd/examples/README.md
@@ -0,0 +1,40 @@
+Zstandard library : usage examples
+==================================
+
+- [Simple compression](simple_compression.c) :
+  Compress a single file.
+  Introduces usage of : `ZSTD_compress()`
+
+- [Simple decompression](simple_decompression.c) :
+  Decompress a single file.
+  Only compatible with simple compression.
+  Result remains in memory.
+  Introduces usage of : `ZSTD_decompress()`
+
+- [Streaming memory usage](streaming_memory_usage.c) :
+  Provides amount of memory used by streaming context
+  Introduces usage of : `ZSTD_sizeof_CStream()`
+
+- [Streaming compression](streaming_compression.c) :
+  Compress a single file.
+  Introduces usage of : `ZSTD_compressStream()`
+
+- [Multiple Streaming compression](multiple_streaming_compression.c) :
+  Compress multiple files in a single command line.
+  Introduces memory usage preservation technique,
+  reducing impact of malloc()/free() and memset() by re-using existing resources.
+
+- [Streaming decompression](streaming_decompression.c) :
+  Decompress a single file compressed by zstd.
+  Compatible with both simple and streaming compression.
+  Result is sent to stdout.
+  Introduces usage of : `ZSTD_decompressStream()`
+
+- [Dictionary compression](dictionary_compression.c) :
+  Compress multiple files using the same dictionary.
+  Introduces usage of : `ZSTD_createCDict()` and `ZSTD_compress_usingCDict()`
+
+- [Dictionary decompression](dictionary_decompression.c) :
+  Decompress multiple files using the same dictionary.
+  Result remains in memory.
+  Introduces usage of : `ZSTD_createDDict()` and `ZSTD_decompress_usingDDict()`
diff --git a/src/zstd/examples/dictionary_compression.c b/src/zstd/examples/dictionary_compression.c
new file mode 100644
index 00000000..97bf8cb5
--- /dev/null
+++ b/src/zstd/examples/dictionary_compression.c
@@ -0,0 +1,157 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#include <stdlib.h>    // malloc, exit
+#include <stdio.h>     // printf
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static off_t fsize_orDie(const char *filename)
+{
+    struct stat st;
+    if (stat(filename, &st) == 0) return st.st_size;
+    /* error */
+    perror(filename);
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(2);
+}
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc");
+    exit(3);
+}
+
+static void* loadFile_orDie(const char* fileName, size_t* size)
+{
+    off_t const buffSize = fsize_orDie(fileName);
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    void* const buffer = malloc_orDie(buffSize);
+    size_t const readSize = fread(buffer, 1, buffSize, inFile);
+    if (readSize != (size_t)buffSize) {
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
+        exit(4);
+    }
+    fclose(inFile);
+    *size = buffSize;
+    return buffer;
+}
+
+static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
+{
+    FILE* const oFile = fopen_orDie(fileName, "wb");
+    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
+    if (wSize != (size_t)buffSize) {
+        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
+        exit(5);
+    }
+    if (fclose(oFile)) {
+        perror(fileName);
+        exit(6);
+    }
+}
+
+/* createDict() :
+   `dictFileName` is supposed to have been created using `zstd --train` */
+static ZSTD_CDict* createCDict_orDie(const char* dictFileName, int cLevel)
+{
+    size_t dictSize;
+    printf("loading dictionary %s \n", dictFileName);
+    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    ZSTD_CDict* const cdict = ZSTD_createCDict(dictBuffer, dictSize, cLevel);
+    if (!cdict) {
+        fprintf(stderr, "ZSTD_createCDict error \n");
+        exit(7);
+    }
+    free(dictBuffer);
+    return cdict;
+}
+
+
+static void compress(const char* fname, const char* oname, const ZSTD_CDict* cdict)
+{
+    size_t fSize;
+    void* const fBuff = loadFile_orDie(fname, &fSize);
+    size_t const cBuffSize = ZSTD_compressBound(fSize);
+    void* const cBuff = malloc_orDie(cBuffSize);
+
+    ZSTD_CCtx* const cctx = ZSTD_createCCtx();
+    if (cctx==NULL) { fprintf(stderr, "ZSTD_createCCtx() error \n"); exit(10); }
+    size_t const cSize = ZSTD_compress_usingCDict(cctx, cBuff, cBuffSize, fBuff, fSize, cdict);
+    if (ZSTD_isError(cSize)) {
+        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
+        exit(7);
+    }
+
+    saveFile_orDie(oname, cBuff, cSize);
+
+    /* success */
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+
+    ZSTD_freeCCtx(cctx);   /* never fails */
+    free(fBuff);
+    free(cBuff);
+}
+
+
+static char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+    int const cLevel = 3;
+
+    if (argc<3) {
+        fprintf(stderr, "wrong arguments\n");
+        fprintf(stderr, "usage:\n");
+        fprintf(stderr, "%s [FILES] dictionary\n", exeName);
+        return 1;
+    }
+
+    /* load dictionary only once */
+    const char* const dictName = argv[argc-1];
+    ZSTD_CDict* const dictPtr = createCDict_orDie(dictName, cLevel);
+
+    int u;
+    for (u=1; u<argc-1; u++) {
+        const char* inFilename = argv[u];
+        char* const outFilename = createOutFilename_orDie(inFilename);
+        compress(inFilename, outFilename, dictPtr);
+        free(outFilename);
+    }
+
+    ZSTD_freeCDict(dictPtr);
+    printf("All %u files compressed. \n", argc-2);
+    return 0;
+}
diff --git a/src/zstd/examples/dictionary_decompression.c b/src/zstd/examples/dictionary_decompression.c
new file mode 100644
index 00000000..07e6e24c
--- /dev/null
+++ b/src/zstd/examples/dictionary_decompression.c
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+
+#include <stdlib.h>    // malloc, exit
+#include <stdio.h>     // printf
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+#define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static off_t fsize_orDie(const char *filename)
+{
+    struct stat st;
+    if (stat(filename, &st) == 0) return st.st_size;
+    /* error */
+    perror(filename);
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(2);
+}
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc");
+    exit(3);
+}
+
+static void* loadFile_orDie(const char* fileName, size_t* size)
+{
+    off_t const buffSize = fsize_orDie(fileName);
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    void* const buffer = malloc_orDie(buffSize);
+    size_t const readSize = fread(buffer, 1, buffSize, inFile);
+    if (readSize != (size_t)buffSize) {
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
+        exit(4);
+    }
+    fclose(inFile);
+    *size = buffSize;
+    return buffer;
+}
+
+/* createDict() :
+   `dictFileName` is supposed to have been created using `zstd --train` */
+static ZSTD_DDict* createDict_orDie(const char* dictFileName)
+{
+    size_t dictSize;
+    printf("loading dictionary %s \n", dictFileName);
+    void* const dictBuffer = loadFile_orDie(dictFileName, &dictSize);
+    ZSTD_DDict* const ddict = ZSTD_createDDict(dictBuffer, dictSize);
+    if (ddict==NULL) { fprintf(stderr, "ZSTD_createDDict error \n"); exit(5); }
+    free(dictBuffer);
+    return ddict;
+}
+
+
+static void decompress(const char* fname, const ZSTD_DDict* ddict)
+{
+    size_t cSize;
+    void* const cBuff = loadFile_orDie(fname, &cSize);
+    unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
+    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
+        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
+        exit(5);
+    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
+        fprintf(stderr, "%s : original size unknown \n", fname);
+        exit(6);
+    }
+
+    void* const rBuff = malloc_orDie((size_t)rSize);
+
+    ZSTD_DCtx* const dctx = ZSTD_createDCtx();
+    if (dctx==NULL) { fprintf(stderr, "ZSTD_createDCtx() error \n"); exit(10); }
+    size_t const dSize = ZSTD_decompress_usingDDict(dctx, rBuff, rSize, cBuff, cSize, ddict);
+    if (dSize != rSize) {
+        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
+        exit(7);
+    }
+
+    /* success */
+    printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
+
+    ZSTD_freeDCtx(dctx);
+    free(rBuff);
+    free(cBuff);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<3) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s [FILES] dictionary\n", exeName);
+        return 1;
+    }
+
+    /* load dictionary only once */
+    const char* const dictName = argv[argc-1];
+    ZSTD_DDict* const dictPtr = createDict_orDie(dictName);
+
+    int u;
+    for (u=1; u<argc-1; u++) decompress(argv[u], dictPtr);
+
+    ZSTD_freeDDict(dictPtr);
+    printf("All %u files correctly decoded (in memory) \n", argc-2);
+    return 0;
+}
diff --git a/src/zstd/examples/multiple_streaming_compression.c b/src/zstd/examples/multiple_streaming_compression.c
new file mode 100644
index 00000000..e395aefb
--- /dev/null
+++ b/src/zstd/examples/multiple_streaming_compression.c
@@ -0,0 +1,165 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/* The objective of this example is to show of to compress multiple successive files
+*  while preserving memory management.
+*  All structures and buffers will be created only once,
+*  and shared across all compression operations */
+
+#include <stdlib.h>    // malloc, exit
+#include <stdio.h>     // fprintf, perror, feof
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#define ZSTD_STATIC_LINKING_ONLY  // streaming API defined as "experimental" for the time being
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc:");
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(4);
+}
+
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(5);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) return 0;
+    /* error */
+    perror("fclose");
+    exit(6);
+}
+
+
+typedef struct {
+    void* buffIn;
+    void* buffOut;
+    size_t buffInSize;
+    size_t buffOutSize;
+    ZSTD_CStream* cstream;
+} resources ;
+
+static resources createResources_orDie()
+{
+    resources ress;
+    ress.buffInSize = ZSTD_CStreamInSize();   /* can always read one full block */
+    ress.buffOutSize= ZSTD_CStreamOutSize();  /* can always flush a full block */
+    ress.buffIn = malloc_orDie(ress.buffInSize);
+    ress.buffOut= malloc_orDie(ress.buffOutSize);
+    ress.cstream = ZSTD_createCStream();
+    if (ress.cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
+    return ress;
+}
+
+static void freeResources(resources ress)
+{
+    ZSTD_freeCStream(ress.cstream);
+    free(ress.buffIn);
+    free(ress.buffOut);
+}
+
+
+static void compressFile_orDie(resources ress, const char* fname, const char* outName, int cLevel)
+{
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+
+    size_t const initResult = ZSTD_initCStream(ress.cstream, cLevel);
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+
+    size_t read, toRead = ress.buffInSize;
+    while( (read = fread_orDie(ress.buffIn, toRead, fin)) ) {
+        ZSTD_inBuffer input = { ress.buffIn, read, 0 };
+        while (input.pos < input.size) {
+            ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+            toRead = ZSTD_compressStream(ress.cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
+            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+            if (toRead > ress.buffInSize) toRead = ress.buffInSize;   /* Safely handle when `buffInSize` is manually changed to a smaller value */
+            fwrite_orDie(ress.buffOut, output.pos, fout);
+        }
+    }
+
+    ZSTD_outBuffer output = { ress.buffOut, ress.buffOutSize, 0 };
+    size_t const remainingToFlush = ZSTD_endStream(ress.cstream, &output);   /* close frame */
+    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
+    fwrite_orDie(ress.buffOut, output.pos, fout);
+
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc<2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE(s)\n", exeName);
+        return 1;
+    }
+
+    resources const ress = createResources_orDie();
+    void* ofnBuffer = NULL;
+    size_t ofnbSize = 0;
+
+    int argNb;
+    for (argNb = 1; argNb < argc; argNb++) {
+        const char* const ifn = argv[argNb];
+        size_t const ifnSize = strlen(ifn);
+        size_t const ofnSize = ifnSize + 5;
+        if (ofnbSize <= ofnSize) {
+            ofnbSize = ofnSize + 16;
+            free(ofnBuffer);
+            ofnBuffer = malloc_orDie(ofnbSize);
+        }
+        memset(ofnBuffer, 0, ofnSize);
+        strcat(ofnBuffer, ifn);
+        strcat(ofnBuffer, ".zst");
+        compressFile_orDie(ress, ifn, ofnBuffer, 7);
+    }
+
+    freeResources(ress);
+    /* success */
+    printf("compressed %i files \n", argc-1);
+
+    return 0;
+}
diff --git a/src/zstd/examples/simple_compression.c b/src/zstd/examples/simple_compression.c
new file mode 100644
index 00000000..9ade424a
--- /dev/null
+++ b/src/zstd/examples/simple_compression.c
@@ -0,0 +1,135 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+
+#include <stdlib.h>    // malloc, free, exit
+#include <stdio.h>     // fprintf, perror, fopen, etc.
+#include <string.h>    // strlen, strcat, memset, strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static off_t fsize_orDie(const char *filename)
+{
+    struct stat st;
+    if (stat(filename, &st) == 0) return st.st_size;
+    /* error */
+    perror(filename);
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(2);
+}
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror(NULL);
+    exit(3);
+}
+
+static void* loadFile_orDie(const char* fileName, size_t* size)
+{
+    off_t const fileSize = fsize_orDie(fileName);
+    size_t const buffSize = (size_t)fileSize;
+    if ((off_t)buffSize < fileSize) {   /* narrowcast overflow */
+        fprintf(stderr, "%s : filesize too large \n", fileName);
+        exit(4);
+    }
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    void* const buffer = malloc_orDie(buffSize);
+    size_t const readSize = fread(buffer, 1, buffSize, inFile);
+    if (readSize != (size_t)buffSize) {
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
+        exit(5);
+    }
+    fclose(inFile);  /* can't fail, read only */
+    *size = buffSize;
+    return buffer;
+}
+
+
+static void saveFile_orDie(const char* fileName, const void* buff, size_t buffSize)
+{
+    FILE* const oFile = fopen_orDie(fileName, "wb");
+    size_t const wSize = fwrite(buff, 1, buffSize, oFile);
+    if (wSize != (size_t)buffSize) {
+        fprintf(stderr, "fwrite: %s : %s \n", fileName, strerror(errno));
+        exit(6);
+    }
+    if (fclose(oFile)) {
+        perror(fileName);
+        exit(7);
+    }
+}
+
+
+static void compress_orDie(const char* fname, const char* oname)
+{
+    size_t fSize;
+    void* const fBuff = loadFile_orDie(fname, &fSize);
+    size_t const cBuffSize = ZSTD_compressBound(fSize);
+    void* const cBuff = malloc_orDie(cBuffSize);
+
+    size_t const cSize = ZSTD_compress(cBuff, cBuffSize, fBuff, fSize, 1);
+    if (ZSTD_isError(cSize)) {
+        fprintf(stderr, "error compressing %s : %s \n", fname, ZSTD_getErrorName(cSize));
+        exit(8);
+    }
+
+    saveFile_orDie(oname, cBuff, cSize);
+
+    /* success */
+    printf("%25s : %6u -> %7u - %s \n", fname, (unsigned)fSize, (unsigned)cSize, oname);
+
+    free(fBuff);
+    free(cBuff);
+}
+
+
+static char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* const outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    const char* const inFilename = argv[1];
+
+    char* const outFilename = createOutFilename_orDie(inFilename);
+    compress_orDie(inFilename, outFilename);
+    free(outFilename);
+    return 0;
+}
diff --git a/src/zstd/examples/simple_decompression.c b/src/zstd/examples/simple_decompression.c
new file mode 100644
index 00000000..c1818a95
--- /dev/null
+++ b/src/zstd/examples/simple_decompression.c
@@ -0,0 +1,110 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+#include <stdlib.h>    // malloc, exit
+#include <stdio.h>     // printf
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#include <sys/stat.h>  // stat
+#define ZSTD_STATIC_LINKING_ONLY   // ZSTD_findDecompressedSize
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static off_t fsize_orDie(const char *filename)
+{
+    struct stat st;
+    if (stat(filename, &st) == 0) return st.st_size;
+    /* error */
+    fprintf(stderr, "stat: %s : %s \n", filename, strerror(errno));
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    fprintf(stderr, "fopen: %s : %s \n", filename, strerror(errno));
+    exit(2);
+}
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size + !size);   /* avoid allocating size of 0 : may return NULL (implementation dependent) */
+    if (buff) return buff;
+    /* error */
+    fprintf(stderr, "malloc: %s \n", strerror(errno));
+    exit(3);
+}
+
+static void* loadFile_orDie(const char* fileName, size_t* size)
+{
+    off_t const buffSize = fsize_orDie(fileName);
+    FILE* const inFile = fopen_orDie(fileName, "rb");
+    void* const buffer = malloc_orDie(buffSize);
+    size_t const readSize = fread(buffer, 1, buffSize, inFile);
+    if (readSize != (size_t)buffSize) {
+        fprintf(stderr, "fread: %s : %s \n", fileName, strerror(errno));
+        exit(4);
+    }
+    fclose(inFile);   /* can't fail (read only) */
+    *size = buffSize;
+    return buffer;
+}
+
+
+static void decompress(const char* fname)
+{
+    size_t cSize;
+    void* const cBuff = loadFile_orDie(fname, &cSize);
+    unsigned long long const rSize = ZSTD_findDecompressedSize(cBuff, cSize);
+    if (rSize==ZSTD_CONTENTSIZE_ERROR) {
+        fprintf(stderr, "%s : it was not compressed by zstd.\n", fname);
+        exit(5);
+    } else if (rSize==ZSTD_CONTENTSIZE_UNKNOWN) {
+        fprintf(stderr,
+                "%s : original size unknown. Use streaming decompression instead.\n", fname);
+        exit(6);
+    }
+
+    void* const rBuff = malloc_orDie((size_t)rSize);
+
+    size_t const dSize = ZSTD_decompress(rBuff, rSize, cBuff, cSize);
+
+    if (dSize != rSize) {
+        fprintf(stderr, "error decoding %s : %s \n", fname, ZSTD_getErrorName(dSize));
+        exit(7);
+    }
+
+    /* success */
+    printf("%25s : %6u -> %7u \n", fname, (unsigned)cSize, (unsigned)rSize);
+
+    free(rBuff);
+    free(cBuff);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    decompress(argv[1]);
+
+    printf("%s correctly decoded (in memory). \n", argv[1]);
+
+    return 0;
+}
diff --git a/src/zstd/examples/streaming_compression.c b/src/zstd/examples/streaming_compression.c
new file mode 100644
index 00000000..f76364d8
--- /dev/null
+++ b/src/zstd/examples/streaming_compression.c
@@ -0,0 +1,131 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#include <stdlib.h>    // malloc, free, exit
+#include <stdio.h>     // fprintf, perror, feof, fopen, etc.
+#include <string.h>    // strlen, memset, strcat
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc:");
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(4);
+}
+
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(5);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) return 0;
+    /* error */
+    perror("fclose");
+    exit(6);
+}
+
+
+static void compressFile_orDie(const char* fname, const char* outName, int cLevel)
+{
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    FILE* const fout = fopen_orDie(outName, "wb");
+    size_t const buffInSize = ZSTD_CStreamInSize();    /* can always read one full block */
+    void*  const buffIn  = malloc_orDie(buffInSize);
+    size_t const buffOutSize = ZSTD_CStreamOutSize();  /* can always flush a full block */
+    void*  const buffOut = malloc_orDie(buffOutSize);
+
+    ZSTD_CStream* const cstream = ZSTD_createCStream();
+    if (cstream==NULL) { fprintf(stderr, "ZSTD_createCStream() error \n"); exit(10); }
+    size_t const initResult = ZSTD_initCStream(cstream, cLevel);
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initCStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+
+    size_t read, toRead = buffInSize;
+    while( (read = fread_orDie(buffIn, toRead, fin)) ) {
+        ZSTD_inBuffer input = { buffIn, read, 0 };
+        while (input.pos < input.size) {
+            ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+            toRead = ZSTD_compressStream(cstream, &output , &input);   /* toRead is guaranteed to be <= ZSTD_CStreamInSize() */
+            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_compressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+            if (toRead > buffInSize) toRead = buffInSize;   /* Safely handle case when `buffInSize` is manually changed to a value < ZSTD_CStreamInSize()*/
+            fwrite_orDie(buffOut, output.pos, fout);
+        }
+    }
+
+    ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+    size_t const remainingToFlush = ZSTD_endStream(cstream, &output);   /* close frame */
+    if (remainingToFlush) { fprintf(stderr, "not fully flushed"); exit(13); }
+    fwrite_orDie(buffOut, output.pos, fout);
+
+    ZSTD_freeCStream(cstream);
+    fclose_orDie(fout);
+    fclose_orDie(fin);
+    free(buffIn);
+    free(buffOut);
+}
+
+
+static const char* createOutFilename_orDie(const char* filename)
+{
+    size_t const inL = strlen(filename);
+    size_t const outL = inL + 5;
+    void* outSpace = malloc_orDie(outL);
+    memset(outSpace, 0, outL);
+    strcat(outSpace, filename);
+    strcat(outSpace, ".zst");
+    return (const char*)outSpace;
+}
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        printf("wrong arguments\n");
+        printf("usage:\n");
+        printf("%s FILE\n", exeName);
+        return 1;
+    }
+
+    const char* const inFilename = argv[1];
+
+    const char* const outFilename = createOutFilename_orDie(inFilename);
+    compressFile_orDie(inFilename, outFilename, 1);
+
+    return 0;
+}
diff --git a/src/zstd/examples/streaming_decompression.c b/src/zstd/examples/streaming_decompression.c
new file mode 100644
index 00000000..504a5e31
--- /dev/null
+++ b/src/zstd/examples/streaming_decompression.c
@@ -0,0 +1,116 @@
+/*
+ * Copyright (c) 2016-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+#include <stdlib.h>    // malloc, exit
+#include <stdio.h>     // fprintf, perror, feof
+#include <string.h>    // strerror
+#include <errno.h>     // errno
+#include <zstd.h>      // presumes zstd library is installed
+
+
+static void* malloc_orDie(size_t size)
+{
+    void* const buff = malloc(size);
+    if (buff) return buff;
+    /* error */
+    perror("malloc:");
+    exit(1);
+}
+
+static FILE* fopen_orDie(const char *filename, const char *instruction)
+{
+    FILE* const inFile = fopen(filename, instruction);
+    if (inFile) return inFile;
+    /* error */
+    perror(filename);
+    exit(3);
+}
+
+static size_t fread_orDie(void* buffer, size_t sizeToRead, FILE* file)
+{
+    size_t const readSize = fread(buffer, 1, sizeToRead, file);
+    if (readSize == sizeToRead) return readSize;   /* good */
+    if (feof(file)) return readSize;   /* good, reached end of file */
+    /* error */
+    perror("fread");
+    exit(4);
+}
+
+static size_t fwrite_orDie(const void* buffer, size_t sizeToWrite, FILE* file)
+{
+    size_t const writtenSize = fwrite(buffer, 1, sizeToWrite, file);
+    if (writtenSize == sizeToWrite) return sizeToWrite;   /* good */
+    /* error */
+    perror("fwrite");
+    exit(5);
+}
+
+static size_t fclose_orDie(FILE* file)
+{
+    if (!fclose(file)) return 0;
+    /* error */
+    perror("fclose");
+    exit(6);
+}
+
+
+static void decompressFile_orDie(const char* fname)
+{
+    FILE* const fin  = fopen_orDie(fname, "rb");
+    size_t const buffInSize = ZSTD_DStreamInSize();
+    void*  const buffIn  = malloc_orDie(buffInSize);
+    FILE* const fout = stdout;
+    size_t const buffOutSize = ZSTD_DStreamOutSize();  /* Guarantee to successfully flush at least one complete compressed block in all circumstances. */
+    void*  const buffOut = malloc_orDie(buffOutSize);
+
+    ZSTD_DStream* const dstream = ZSTD_createDStream();
+    if (dstream==NULL) { fprintf(stderr, "ZSTD_createDStream() error \n"); exit(10); }
+
+    /* In more complex scenarios, a file may consist of multiple appended frames (ex : pzstd).
+    *  The following example decompresses only the first frame.
+    *  It is compatible with other provided streaming examples */
+    size_t const initResult = ZSTD_initDStream(dstream);
+    if (ZSTD_isError(initResult)) { fprintf(stderr, "ZSTD_initDStream() error : %s \n", ZSTD_getErrorName(initResult)); exit(11); }
+    size_t read, toRead = initResult;
+    while ( (read = fread_orDie(buffIn, toRead, fin)) ) {
+        ZSTD_inBuffer input = { buffIn, read, 0 };
+        while (input.pos < input.size) {
+            ZSTD_outBuffer output = { buffOut, buffOutSize, 0 };
+            toRead = ZSTD_decompressStream(dstream, &output , &input);  /* toRead : size of next compressed block */
+            if (ZSTD_isError(toRead)) { fprintf(stderr, "ZSTD_decompressStream() error : %s \n", ZSTD_getErrorName(toRead)); exit(12); }
+            fwrite_orDie(buffOut, output.pos, fout);
+        }
+    }
+
+    ZSTD_freeDStream(dstream);
+    fclose_orDie(fin);
+    fclose_orDie(fout);
+    free(buffIn);
+    free(buffOut);
+}
+
+
+int main(int argc, const char** argv)
+{
+    const char* const exeName = argv[0];
+
+    if (argc!=2) {
+        fprintf(stderr, "wrong arguments\n");
+        fprintf(stderr, "usage:\n");
+        fprintf(stderr, "%s FILE\n", exeName);
+        return 1;
+    }
+
+    const char* const inFilename = argv[1];
+
+    decompressFile_orDie(inFilename);
+    return 0;
+}
diff --git a/src/zstd/examples/streaming_memory_usage.c b/src/zstd/examples/streaming_memory_usage.c
new file mode 100644
index 00000000..b056c2a5
--- /dev/null
+++ b/src/zstd/examples/streaming_memory_usage.c
@@ -0,0 +1,149 @@
+/*
+ * Copyright (c) 2017-present, Yann Collet, Facebook, Inc.
+ * All rights reserved.
+ *
+ * This source code is licensed under both the BSD-style license (found in the
+ * LICENSE file in the root directory of this source tree) and the GPLv2 (found
+ * in the COPYING file in the root directory of this source tree).
+ * You may select, at your option, one of the above-listed licenses.
+ */
+
+
+/*===   Tuning parameter   ===*/
+#ifndef MAX_TESTED_LEVEL
+#define MAX_TESTED_LEVEL 12
+#endif
+
+
+/*===   Dependencies   ===*/
+#include <stdio.h>   /* printf */
+#define ZSTD_STATIC_LINKING_ONLY
+#include "zstd.h"
+
+
+/*===   functions   ===*/
+
+/*! readU32FromChar() :
+    @return : unsigned integer value read from input in `char` format
+    allows and interprets K, KB, KiB, M, MB and MiB suffix.
+    Will also modify `*stringPtr`, advancing it to position where it stopped reading.
+    Note : function result can overflow if digit string > MAX_UINT */
+static unsigned readU32FromChar(const char** stringPtr)
+{
+    unsigned result = 0;
+    while ((**stringPtr >='0') && (**stringPtr <='9'))
+        result *= 10, result += **stringPtr - '0', (*stringPtr)++ ;
+    if ((**stringPtr=='K') || (**stringPtr=='M')) {
+        result <<= 10;
+        if (**stringPtr=='M') result <<= 10;
+        (*stringPtr)++ ;
+        if (**stringPtr=='i') (*stringPtr)++;
+        if (**stringPtr=='B') (*stringPtr)++;
+    }
+    return result;
+}
+
+
+int main(int argc, char const *argv[]) {
+
+    printf("\n Zstandard (v%u) memory usage for streaming contexts : \n\n", ZSTD_versionNumber());
+
+    unsigned wLog = 0;
+    if (argc > 1) {
+        const char* valStr = argv[1];
+        wLog = readU32FromChar(&valStr);
+    }
+
+    int compressionLevel;
+    for (compressionLevel = 1; compressionLevel <= MAX_TESTED_LEVEL; compressionLevel++) {
+#define INPUT_SIZE 5
+#define COMPRESSED_SIZE 128
+        char const dataToCompress[INPUT_SIZE] = "abcde";
+        char compressedData[COMPRESSED_SIZE];
+        char decompressedData[INPUT_SIZE];
+        ZSTD_CStream* const cstream = ZSTD_createCStream();
+        if (cstream==NULL) {
+            printf("Level %i : ZSTD_CStream Memory allocation failure \n", compressionLevel);
+            return 1;
+        }
+
+        /* forces compressor to use maximum memory size for given compression level,
+         * by not providing any information on input size */
+        ZSTD_parameters params = ZSTD_getParams(compressionLevel, 0, 0);
+        if (wLog) { /* special mode : specific wLog */
+            printf("Using custom compression parameter : level 1 + wLog=%u \n", wLog);
+            params = ZSTD_getParams(1, 1 << wLog, 0);
+            size_t const error = ZSTD_initCStream_advanced(cstream, NULL, 0, params, 0);
+            if (ZSTD_isError(error)) {
+                printf("ZSTD_initCStream_advanced error : %s \n", ZSTD_getErrorName(error));
+                return 1;
+            }
+        } else {
+            size_t const error = ZSTD_initCStream(cstream, compressionLevel);
+            if (ZSTD_isError(error)) {
+                printf("ZSTD_initCStream error : %s \n", ZSTD_getErrorName(error));
+                return 1;
+            }
+        }
+
+        size_t compressedSize;
+        {   ZSTD_inBuffer inBuff = { dataToCompress, sizeof(dataToCompress), 0 };
+            ZSTD_outBuffer outBuff = { compressedData, sizeof(compressedData), 0 };
+            size_t const cError = ZSTD_compressStream(cstream, &outBuff, &inBuff);
+            if (ZSTD_isError(cError)) {
+                printf("ZSTD_compressStream error : %s \n", ZSTD_getErrorName(cError));
+                return 1;
+            }
+            size_t const fError = ZSTD_endStream(cstream, &outBuff);
+            if (ZSTD_isError(fError)) {
+                printf("ZSTD_endStream error : %s \n", ZSTD_getErrorName(fError));
+                return 1;
+            }
+            compressedSize = outBuff.pos;
+        }
+
+        ZSTD_DStream* dstream = ZSTD_createDStream();
+        if (dstream==NULL) {
+            printf("Level %i : ZSTD_DStream Memory allocation failure \n", compressionLevel);
+            return 1;
+        }
+        {   size_t const error = ZSTD_initDStream(dstream);
+            if (ZSTD_isError(error)) {
+                printf("ZSTD_initDStream error : %s \n", ZSTD_getErrorName(error));
+                return 1;
+            }
+        }
+        /* forces decompressor to use maximum memory size, as decompressed size is not known */
+        {   ZSTD_inBuffer inBuff = { compressedData, compressedSize, 0 };
+            ZSTD_outBuffer outBuff = { decompressedData, sizeof(decompressedData), 0 };
+            size_t const dResult = ZSTD_decompressStream(dstream, &outBuff, &inBuff);
+            if (ZSTD_isError(dResult)) {
+                printf("ZSTD_decompressStream error : %s \n", ZSTD_getErrorName(dResult));
+                return 1;
+            }
+            if (dResult != 0) {
+                printf("ZSTD_decompressStream error : unfinished decompression \n");
+                return 1;
+            }
+            if (outBuff.pos != sizeof(dataToCompress)) {
+                printf("ZSTD_decompressStream error : incorrect decompression \n");
+                return 1;
+            }
+        }
+
+        size_t const cstreamSize = ZSTD_sizeof_CStream(cstream);
+        size_t const cstreamEstimatedSize = wLog ?
+                ZSTD_estimateCStreamSize_usingCParams(params.cParams) :
+                ZSTD_estimateCStreamSize(compressionLevel);
+        size_t const dstreamSize = ZSTD_sizeof_DStream(dstream);
+
+        printf("Level %2i : Compression Mem = %5u KB (estimated : %5u KB) ; Decompression Mem = %4u KB \n",
+                compressionLevel,
+                (unsigned)(cstreamSize>>10), (unsigned)(cstreamEstimatedSize>>10), (unsigned)(dstreamSize>>10));
+
+        ZSTD_freeDStream(dstream);
+        ZSTD_freeCStream(cstream);
+        if (wLog) break;  /* single test */
+    }
+    return 0;
+}