Adding upstream version 1:115.7.0.upstream/1%115.7.0upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

14 files changed, 1227 insertions, 0 deletions

diff --git a/comm/third_party/botan/src/lib/compression/bzip2/bzip2.cpp b/comm/third_party/botan/src/lib/compression/bzip2/bzip2.cpp
new file mode 100644
index 0000000000..c9dfc2ce82
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/bzip2/bzip2.cpp
@@ -0,0 +1,110 @@
+/*
+* Bzip2 Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007,2014 Jack Lloyd
+*     2006 Matt Johnston
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/bzip2.h>
+#include <botan/exceptn.h>
+#include <botan/internal/compress_utils.h>
+
+#define BZ_NO_STDIO
+#include <bzlib.h>
+
+namespace Botan {
+
+namespace {
+
+class Bzip2_Stream : public Zlib_Style_Stream<bz_stream, char>
+   {
+   public:
+      Bzip2_Stream()
+         {
+         streamp()->opaque = alloc();
+         streamp()->bzalloc = Compression_Alloc_Info::malloc<int>;
+         streamp()->bzfree = Compression_Alloc_Info::free;
+         }
+
+      uint32_t run_flag() const override { return BZ_RUN; }
+      uint32_t flush_flag() const override { return BZ_FLUSH; }
+      uint32_t finish_flag() const override { return BZ_FINISH; }
+   };
+
+class Bzip2_Compression_Stream final : public Bzip2_Stream
+   {
+   public:
+      explicit Bzip2_Compression_Stream(size_t block_size)
+         {
+         /*
+         * Defaults to 900k blocks as the computation cost of
+         * compression is not overly affected by the size, though
+         * more memory is required.
+         */
+         if(block_size == 0 || block_size >= 9)
+            block_size = 9;
+
+         int rc = BZ2_bzCompressInit(streamp(), block_size, 0, 0);
+
+         if(rc != BZ_OK)
+            throw Compression_Error("BZ2_bzCompressInit", ErrorType::Bzip2Error, rc);
+         }
+
+      ~Bzip2_Compression_Stream()
+         {
+         BZ2_bzCompressEnd(streamp());
+         }
+
+      bool run(uint32_t flags) override
+         {
+         int rc = BZ2_bzCompress(streamp(), flags);
+
+         if(rc < 0)
+            throw Compression_Error("BZ2_bzCompress", ErrorType::Bzip2Error, rc);
+
+         return (rc == BZ_STREAM_END);
+         }
+   };
+
+class Bzip2_Decompression_Stream final : public Bzip2_Stream
+   {
+   public:
+      Bzip2_Decompression_Stream()
+         {
+         int rc = BZ2_bzDecompressInit(streamp(), 0, 0);
+
+         if(rc != BZ_OK)
+            throw Compression_Error("BZ2_bzDecompressInit", ErrorType::Bzip2Error, rc);
+         }
+
+      ~Bzip2_Decompression_Stream()
+         {
+         BZ2_bzDecompressEnd(streamp());
+         }
+
+      bool run(uint32_t) override
+         {
+         int rc = BZ2_bzDecompress(streamp());
+
+         if(rc != BZ_OK && rc != BZ_STREAM_END)
+            throw Compression_Error("BZ2_bzDecompress", ErrorType::Bzip2Error, rc);
+
+         return (rc == BZ_STREAM_END);
+         }
+   };
+
+}
+
+Compression_Stream* Bzip2_Compression::make_stream(size_t comp_level) const
+   {
+   return new Bzip2_Compression_Stream(comp_level);
+   }
+
+Compression_Stream* Bzip2_Decompression::make_stream() const
+   {
+   return new Bzip2_Decompression_Stream;
+   }
+
+}
diff --git a/comm/third_party/botan/src/lib/compression/bzip2/bzip2.h b/comm/third_party/botan/src/lib/compression/bzip2/bzip2.h
new file mode 100644
index 0000000000..e056d55021
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/bzip2/bzip2.h
@@ -0,0 +1,40 @@
+/*
+* Bzip2 Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007,2014 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_BZIP2_H_
+#define BOTAN_BZIP2_H_
+
+#include <botan/compression.h>
+
+namespace Botan {
+
+/**
+* Bzip2 Compression
+*/
+class BOTAN_PUBLIC_API(2,0) Bzip2_Compression final : public Stream_Compression
+   {
+   public:
+      std::string name() const override { return "Bzip2_Compression"; }
+   private:
+      Compression_Stream* make_stream(size_t comp_level) const override;
+   };
+
+/**
+* Bzip2 Deccompression
+*/
+class BOTAN_PUBLIC_API(2,0) Bzip2_Decompression final : public Stream_Decompression
+   {
+   public:
+      std::string name() const override { return "Bzip2_Decompression"; }
+   private:
+      Compression_Stream* make_stream() const override;
+   };
+
+}
+
+#endif
diff --git a/comm/third_party/botan/src/lib/compression/bzip2/info.txt b/comm/third_party/botan/src/lib/compression/bzip2/info.txt
new file mode 100644
index 0000000000..8826df2f96
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/bzip2/info.txt
@@ -0,0 +1,9 @@
+<defines>
+BZIP2 -> 20160412
+</defines>
+
+load_on vendor
+
+<libs>
+all -> bz2
+</libs>
diff --git a/comm/third_party/botan/src/lib/compression/compress_utils.cpp b/comm/third_party/botan/src/lib/compression/compress_utils.cpp
new file mode 100644
index 0000000000..f49a0ede12
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/compress_utils.cpp
@@ -0,0 +1,196 @@
+/*
+* Compression Utils
+* (C) 2014,2016 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/internal/compress_utils.h>
+#include <botan/exceptn.h>
+#include <cstdlib>
+
+namespace Botan {
+
+void* Compression_Alloc_Info::do_malloc(size_t n, size_t size)
+   {
+   // TODO maximum length check here?
+   void* ptr = std::calloc(n, size);
+
+   /*
+   * Return null rather than throwing here as we are being called by a
+   * C library and it may not be possible for an exception to unwind
+   * the call stack from here. The compression library is expecting a
+   * function written in C and a null return on error, which it will
+   * send upwards to the compression wrappers.
+   */
+
+   if(ptr)
+      {
+      m_current_allocs[ptr] = n * size;
+      }
+
+   return ptr;
+   }
+
+void Compression_Alloc_Info::do_free(void* ptr)
+   {
+   if(ptr)
+      {
+      auto i = m_current_allocs.find(ptr);
+
+      if(i == m_current_allocs.end())
+         throw Internal_Error("Compression_Alloc_Info::free got pointer not allocated by us");
+
+      secure_scrub_memory(ptr, i->second);
+      std::free(ptr);
+      m_current_allocs.erase(i);
+      }
+   }
+
+void Stream_Compression::clear()
+   {
+   m_stream.reset();
+   }
+
+void Stream_Compression::start(size_t level)
+   {
+   m_stream.reset(make_stream(level));
+   }
+
+void Stream_Compression::process(secure_vector<uint8_t>& buf, size_t offset, uint32_t flags)
+   {
+   BOTAN_ASSERT(m_stream, "Initialized");
+   BOTAN_ASSERT(buf.size() >= offset, "Offset is sane");
+
+   // bzip doesn't like being called with no input and BZ_RUN
+   if(buf.size() == offset && flags == m_stream->run_flag())
+      {
+      return;
+      }
+
+   if(m_buffer.size() < buf.size() + offset)
+      m_buffer.resize(buf.size() + offset);
+
+   // If the output buffer has zero length, .data() might return nullptr. This would
+   // make some compression algorithms (notably those provided by zlib) fail.
+   // Any small positive value works fine, but we choose 32 as it is the smallest power
+   // of two that is large enough to hold all the headers and trailers of the common
+   // formats, preventing further resizings to make room for output data.
+   if(m_buffer.size() == 0)
+      m_buffer.resize(32);
+
+   m_stream->next_in(buf.data() + offset, buf.size() - offset);
+   m_stream->next_out(m_buffer.data() + offset, m_buffer.size() - offset);
+
+   while(true)
+      {
+      const bool stream_end = m_stream->run(flags);
+
+      if(stream_end)
+         {
+         BOTAN_ASSERT(m_stream->avail_in() == 0, "After stream is done, no input remains to be processed");
+         m_buffer.resize(m_buffer.size() - m_stream->avail_out());
+         break;
+         }
+      else if(m_stream->avail_out() == 0)
+         {
+         const size_t added = 8 + m_buffer.size();
+         m_buffer.resize(m_buffer.size() + added);
+         m_stream->next_out(m_buffer.data() + m_buffer.size() - added, added);
+         }
+      else if(m_stream->avail_in() == 0)
+         {
+         m_buffer.resize(m_buffer.size() - m_stream->avail_out());
+         break;
+         }
+      }
+
+   copy_mem(m_buffer.data(), buf.data(), offset);
+   buf.swap(m_buffer);
+   }
+
+void Stream_Compression::update(secure_vector<uint8_t>& buf, size_t offset, bool flush)
+   {
+   BOTAN_ASSERT(m_stream, "Initialized");
+   process(buf, offset, flush ? m_stream->flush_flag() : m_stream->run_flag());
+   }
+
+void Stream_Compression::finish(secure_vector<uint8_t>& buf, size_t offset)
+   {
+   BOTAN_ASSERT(m_stream, "Initialized");
+   process(buf, offset, m_stream->finish_flag());
+   clear();
+   }
+
+void Stream_Decompression::clear()
+   {
+   m_stream.reset();
+   }
+
+void Stream_Decompression::start()
+   {
+   m_stream.reset(make_stream());
+   }
+
+void Stream_Decompression::process(secure_vector<uint8_t>& buf, size_t offset, uint32_t flags)
+   {
+   BOTAN_ASSERT(m_stream, "Initialized");
+   BOTAN_ASSERT(buf.size() >= offset, "Offset is sane");
+
+   if(m_buffer.size() < buf.size() + offset)
+      m_buffer.resize(buf.size() + offset);
+
+   m_stream->next_in(buf.data() + offset, buf.size() - offset);
+   m_stream->next_out(m_buffer.data() + offset, m_buffer.size() - offset);
+
+   while(true)
+      {
+      const bool stream_end = m_stream->run(flags);
+
+      if(stream_end)
+         {
+         if(m_stream->avail_in() == 0) // all data consumed?
+            {
+            m_buffer.resize(m_buffer.size() - m_stream->avail_out());
+            clear();
+            break;
+            }
+
+         // More data follows: try to process as a following stream
+         const size_t read = (buf.size() - offset) - m_stream->avail_in();
+         start();
+         m_stream->next_in(buf.data() + offset + read, buf.size() - offset - read);
+         }
+
+      if(m_stream->avail_out() == 0)
+         {
+         const size_t added = 8 + m_buffer.size();
+         m_buffer.resize(m_buffer.size() + added);
+         m_stream->next_out(m_buffer.data() + m_buffer.size() - added, added);
+         }
+      else if(m_stream->avail_in() == 0)
+         {
+         m_buffer.resize(m_buffer.size() - m_stream->avail_out());
+         break;
+         }
+      }
+
+   copy_mem(m_buffer.data(), buf.data(), offset);
+   buf.swap(m_buffer);
+   }
+
+void Stream_Decompression::update(secure_vector<uint8_t>& buf, size_t offset)
+   {
+   process(buf, offset, m_stream->run_flag());
+   }
+
+void Stream_Decompression::finish(secure_vector<uint8_t>& buf, size_t offset)
+   {
+   if(buf.size() != offset || m_stream.get())
+      process(buf, offset, m_stream->finish_flag());
+
+   if(m_stream.get())
+      throw Invalid_State(name() + " finished but not at stream end");
+   }
+
+}
diff --git a/comm/third_party/botan/src/lib/compression/compress_utils.h b/comm/third_party/botan/src/lib/compression/compress_utils.h
new file mode 100644
index 0000000000..396fc47bed
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/compress_utils.h
@@ -0,0 +1,89 @@
+/*
+* Compression utility header
+* (C) 2014 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_COMPRESSION_UTILS_H_
+#define BOTAN_COMPRESSION_UTILS_H_
+
+#include <botan/compression.h>
+#include <memory>
+#include <unordered_map>
+
+namespace Botan {
+
+/*
+* Allocation Size Tracking Helper for Zlib/Bzlib/LZMA
+*/
+class Compression_Alloc_Info final
+   {
+   public:
+      template<typename T>
+      static void* malloc(void* self, T n, T size)
+         {
+         return static_cast<Compression_Alloc_Info*>(self)->do_malloc(n, size);
+         }
+
+      static void free(void* self, void* ptr)
+         {
+         static_cast<Compression_Alloc_Info*>(self)->do_free(ptr);
+         }
+
+   private:
+      void* do_malloc(size_t n, size_t size);
+      void do_free(void* ptr);
+
+      std::unordered_map<void*, size_t> m_current_allocs;
+   };
+
+/**
+* Wrapper for Zlib/Bzlib/LZMA stream types
+*/
+template<typename Stream, typename ByteType>
+class Zlib_Style_Stream : public Compression_Stream
+   {
+   public:
+      void next_in(uint8_t* b, size_t len) override
+         {
+         m_stream.next_in = reinterpret_cast<ByteType*>(b);
+         m_stream.avail_in = len;
+         }
+
+      void next_out(uint8_t* b, size_t len) override
+         {
+         m_stream.next_out = reinterpret_cast<ByteType*>(b);
+         m_stream.avail_out = len;
+         }
+
+      size_t avail_in() const override { return m_stream.avail_in; }
+
+      size_t avail_out() const override { return m_stream.avail_out; }
+
+      Zlib_Style_Stream()
+         {
+         clear_mem(&m_stream, 1);
+         m_allocs.reset(new Compression_Alloc_Info);
+         }
+
+      ~Zlib_Style_Stream()
+         {
+         clear_mem(&m_stream, 1);
+         m_allocs.reset();
+         }
+
+   protected:
+      typedef Stream stream_t;
+
+      stream_t* streamp() { return &m_stream; }
+
+      Compression_Alloc_Info* alloc() { return m_allocs.get(); }
+   private:
+      stream_t m_stream;
+      std::unique_ptr<Compression_Alloc_Info> m_allocs;
+   };
+
+}
+
+#endif
diff --git a/comm/third_party/botan/src/lib/compression/compression.cpp b/comm/third_party/botan/src/lib/compression/compression.cpp
new file mode 100644
index 0000000000..361bf7dd3c
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/compression.cpp
@@ -0,0 +1,116 @@
+/*
+* Compression Factory
+* (C) 2014,2016 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/compression.h>
+#include <botan/mem_ops.h>
+#include <botan/exceptn.h>
+#include <cstdlib>
+
+#if defined(BOTAN_HAS_ZLIB)
+  #include <botan/zlib.h>
+#endif
+
+#if defined(BOTAN_HAS_BZIP2)
+  #include <botan/bzip2.h>
+#endif
+
+#if defined(BOTAN_HAS_LZMA)
+  #include <botan/lzma.h>
+#endif
+
+namespace Botan {
+
+Compression_Algorithm* make_compressor(const std::string& name)
+   {
+#if defined(BOTAN_HAS_ZLIB)
+   if(name == "Zlib" || name == "zlib")
+      return new Zlib_Compression;
+   if(name == "Gzip" || name == "gzip" || name == "gz")
+      return new Gzip_Compression;
+   if(name == "Deflate" || name == "deflate")
+      return new Deflate_Compression;
+#endif
+
+#if defined(BOTAN_HAS_BZIP2)
+   if(name == "bzip2" || name == "bz2" || name == "Bzip2")
+      return new Bzip2_Compression;
+#endif
+
+#if defined(BOTAN_HAS_LZMA)
+   if(name == "lzma" || name == "xz" || name == "LZMA")
+      return new LZMA_Compression;
+#endif
+
+   BOTAN_UNUSED(name);
+   return nullptr;
+   }
+
+//static
+std::unique_ptr<Compression_Algorithm>
+Compression_Algorithm::create(const std::string& algo)
+   {
+   std::unique_ptr<Compression_Algorithm> compressor(make_compressor(algo));
+   return compressor;
+   }
+
+//static
+std::unique_ptr<Compression_Algorithm>
+Compression_Algorithm::create_or_throw(const std::string& algo)
+   {
+   if(auto compressor = Compression_Algorithm::create(algo))
+      {
+      return compressor;
+      }
+   throw Lookup_Error("Compression", algo, "");
+   }
+
+Decompression_Algorithm* make_decompressor(const std::string& name)
+   {
+#if defined(BOTAN_HAS_ZLIB)
+   if(name == "Zlib" || name == "zlib")
+      return new Zlib_Decompression;
+   if(name == "Gzip" || name == "gzip" || name == "gz")
+      return new Gzip_Decompression;
+   if(name == "Deflate" || name == "deflate")
+      return new Deflate_Decompression;
+#endif
+
+#if defined(BOTAN_HAS_BZIP2)
+   if(name == "bzip2" || name == "bz2" || name == "Bzip2")
+      return new Bzip2_Decompression;
+#endif
+
+#if defined(BOTAN_HAS_LZMA)
+   if(name == "lzma" || name == "xz" || name == "LZMA")
+      return new LZMA_Decompression;
+#endif
+
+   BOTAN_UNUSED(name);
+   return nullptr;
+   }
+
+//static
+std::unique_ptr<Decompression_Algorithm>
+Decompression_Algorithm::create(const std::string& algo)
+   {
+   std::unique_ptr<Decompression_Algorithm> decompressor(make_decompressor(algo));
+   return decompressor;
+   }
+
+//static
+std::unique_ptr<Decompression_Algorithm>
+Decompression_Algorithm::create_or_throw(const std::string& algo)
+   {
+   if(auto decompressor = Decompression_Algorithm::create(algo))
+      {
+      return decompressor;
+      }
+   throw Lookup_Error("Decompression", algo, "");
+   }
+
+}
+
diff --git a/comm/third_party/botan/src/lib/compression/compression.h b/comm/third_party/botan/src/lib/compression/compression.h
new file mode 100644
index 0000000000..217fae623a
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/compression.h
@@ -0,0 +1,238 @@
+/*
+* Compression Transform
+* (C) 2014 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_COMPRESSION_TRANSFORM_H_
+#define BOTAN_COMPRESSION_TRANSFORM_H_
+
+#include <botan/secmem.h>
+#include <botan/exceptn.h>
+#include <string>
+
+namespace Botan {
+
+/**
+* Interface for a compression algorithm.
+*/
+class BOTAN_PUBLIC_API(2,0) Compression_Algorithm
+   {
+   public:
+      /**
+      * Create an instance based on a name, or return null if the
+      * algo combination cannot be found.
+      */
+      static std::unique_ptr<Compression_Algorithm>
+         create(const std::string& algo_spec);
+
+      /**
+      * Create an instance based on a name
+      * @param algo_spec algorithm name
+      * Throws Lookup_Error if not found.
+      */
+      static std::unique_ptr<Compression_Algorithm>
+         create_or_throw(const std::string& algo_spec);
+
+      /**
+      * Begin compressing. Most compression algorithms offer a tunable
+      * time/compression tradeoff parameter generally represented by
+      * an integer in the range of 1 to 9.
+      *
+      * If 0 or a value out of range is provided, a compression algorithm
+      * specific default is used.
+      */
+      virtual void start(size_t comp_level = 0) = 0;
+
+      /**
+      * Process some data.
+      * @param buf in/out parameter which will possibly be resized or swapped
+      * @param offset an offset into blocks to begin processing
+      * @param flush if true the compressor will be told to flush state
+      */
+      virtual void update(secure_vector<uint8_t>& buf, size_t offset = 0, bool flush = false) = 0;
+
+      /**
+      * Finish compressing
+      *
+      * @param final_block in/out parameter
+      * @param offset an offset into final_block to begin processing
+      */
+      virtual void finish(secure_vector<uint8_t>& final_block, size_t offset = 0) = 0;
+
+      /**
+      * @return name of the compression algorithm
+      */
+      virtual std::string name() const = 0;
+
+      /**
+      * Reset the state and abort the current message; start can be
+      * called again to process a new message.
+      */
+      virtual void clear() = 0;
+
+      virtual ~Compression_Algorithm() = default;
+   };
+
+/*
+* Interface for a decompression algorithm.
+*/
+class BOTAN_PUBLIC_API(2,0) Decompression_Algorithm
+   {
+   public:
+      /**
+      * Create an instance based on a name, or return null if the
+      * algo combination cannot be found.
+      */
+      static std::unique_ptr<Decompression_Algorithm>
+         create(const std::string& algo_spec);
+
+      /**
+      * Create an instance based on a name
+      * @param algo_spec algorithm name
+      * Throws Lookup_Error if not found.
+      */
+      static std::unique_ptr<Decompression_Algorithm>
+         create_or_throw(const std::string& algo_spec);
+
+      /**
+      * Begin decompressing.
+      * Decompression does not support levels, as compression does.
+      */
+      virtual void start() = 0;
+
+      /**
+      * Process some data.
+      * @param buf in/out parameter which will possibly be resized or swapped
+      * @param offset an offset into blocks to begin processing
+      */
+      virtual void update(secure_vector<uint8_t>& buf, size_t offset = 0) = 0;
+
+      /**
+      * Finish decompressing
+      *
+      * @param final_block in/out parameter
+      * @param offset an offset into final_block to begin processing
+      */
+      virtual void finish(secure_vector<uint8_t>& final_block, size_t offset = 0) = 0;
+
+      /**
+      * @return name of the decompression algorithm
+      */
+      virtual std::string name() const = 0;
+
+      /**
+      * Reset the state and abort the current message; start can be
+      * called again to process a new message.
+      */
+      virtual void clear() = 0;
+
+      virtual ~Decompression_Algorithm() = default;
+   };
+
+BOTAN_PUBLIC_API(2,0) Compression_Algorithm* make_compressor(const std::string& type);
+BOTAN_PUBLIC_API(2,0) Decompression_Algorithm* make_decompressor(const std::string& type);
+
+/**
+* An error that occurred during compression (or decompression)
+*/
+class BOTAN_PUBLIC_API(2,9) Compression_Error : public Exception
+   {
+   public:
+
+      /**
+      * @param func_name the name of the compression API that was called
+      * (eg "BZ2_bzCompressInit" or "lzma_code")
+      * @param type what library this came from
+      * @param rc the error return code from the compression API. The
+      * interpretation of this value will depend on the library.
+      */
+     Compression_Error(const char* func_name, ErrorType type, int rc) :
+        Exception("Compression API " + std::string(func_name) +
+                  " failed with return code " + std::to_string(rc)),
+        m_type(type),
+        m_rc(rc)
+         {}
+
+      ErrorType error_type() const noexcept override { return m_type; }
+
+      int error_code() const noexcept override { return m_rc; }
+
+   private:
+      ErrorType m_type;
+      int m_rc;
+   };
+
+/**
+* Adapts a zlib style API
+*/
+class Compression_Stream
+   {
+   public:
+      virtual ~Compression_Stream() = default;
+
+      virtual void next_in(uint8_t* b, size_t len) = 0;
+
+      virtual void next_out(uint8_t* b, size_t len) = 0;
+
+      virtual size_t avail_in() const = 0;
+
+      virtual size_t avail_out() const = 0;
+
+      virtual uint32_t run_flag() const = 0;
+      virtual uint32_t flush_flag() const = 0;
+      virtual uint32_t finish_flag() const = 0;
+
+      virtual bool run(uint32_t flags) = 0;
+   };
+
+/**
+* Used to implement compression using Compression_Stream
+*/
+class Stream_Compression : public Compression_Algorithm
+   {
+   public:
+      void update(secure_vector<uint8_t>& buf, size_t offset, bool flush) final override;
+
+      void finish(secure_vector<uint8_t>& buf, size_t offset) final override;
+
+      void clear() final override;
+
+   private:
+      void start(size_t level) final override;
+
+      void process(secure_vector<uint8_t>& buf, size_t offset, uint32_t flags);
+
+      virtual Compression_Stream* make_stream(size_t level) const = 0;
+
+      secure_vector<uint8_t> m_buffer;
+      std::unique_ptr<Compression_Stream> m_stream;
+   };
+
+/**
+* FIXME add doc
+*/
+class Stream_Decompression : public Decompression_Algorithm
+   {
+   public:
+      void update(secure_vector<uint8_t>& buf, size_t offset) final override;
+
+      void finish(secure_vector<uint8_t>& buf, size_t offset) final override;
+
+      void clear() final override;
+
+   private:
+      void start() final override;
+
+      void process(secure_vector<uint8_t>& buf, size_t offset, uint32_t flags);
+
+      virtual Compression_Stream* make_stream() const = 0;
+
+      secure_vector<uint8_t> m_buffer;
+      std::unique_ptr<Compression_Stream> m_stream;
+   };
+
+}
+
+#endif
diff --git a/comm/third_party/botan/src/lib/compression/info.txt b/comm/third_party/botan/src/lib/compression/info.txt
new file mode 100644
index 0000000000..fade2e51aa
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/info.txt
@@ -0,0 +1,11 @@
+<defines>
+COMPRESSION -> 20141117
+</defines>
+
+<header:internal>
+compress_utils.h
+</header:internal>
+
+<header:public>
+compression.h
+</header:public>
diff --git a/comm/third_party/botan/src/lib/compression/lzma/info.txt b/comm/third_party/botan/src/lib/compression/lzma/info.txt
new file mode 100644
index 0000000000..477a7b7951
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/lzma/info.txt
@@ -0,0 +1,9 @@
+<defines>
+LZMA -> 20160412
+</defines>
+
+load_on vendor
+
+<libs>
+all -> lzma
+</libs>
diff --git a/comm/third_party/botan/src/lib/compression/lzma/lzma.cpp b/comm/third_party/botan/src/lib/compression/lzma/lzma.cpp
new file mode 100644
index 0000000000..73bb9eb89f
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/lzma/lzma.cpp
@@ -0,0 +1,95 @@
+/*
+* Lzma Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007,2014 Jack Lloyd
+*     2006 Matt Johnston
+*     2012 Vojtech Kral
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/lzma.h>
+#include <botan/internal/compress_utils.h>
+#include <botan/exceptn.h>
+#include <lzma.h>
+
+namespace Botan {
+
+namespace {
+
+class LZMA_Stream : public Zlib_Style_Stream<lzma_stream, uint8_t>
+   {
+   public:
+      LZMA_Stream()
+         {
+         m_allocator.opaque = alloc();
+         m_allocator.alloc = Compression_Alloc_Info::malloc<size_t>;
+         m_allocator.free = Compression_Alloc_Info::free;
+         streamp()->allocator = &m_allocator;
+         }
+
+      ~LZMA_Stream()
+         {
+         ::lzma_end(streamp());
+         }
+
+      bool run(uint32_t flags) override
+         {
+         lzma_ret rc = ::lzma_code(streamp(), static_cast<lzma_action>(flags));
+
+         if(rc != LZMA_OK && rc != LZMA_STREAM_END)
+            throw Compression_Error("lzma_code", ErrorType::LzmaError, rc);
+
+         return (rc == LZMA_STREAM_END);
+         }
+
+      uint32_t run_flag() const override { return LZMA_RUN; }
+      uint32_t flush_flag() const override { return LZMA_FULL_FLUSH; }
+      uint32_t finish_flag() const override { return LZMA_FINISH; }
+   private:
+      ::lzma_allocator m_allocator;
+   };
+
+class LZMA_Compression_Stream final : public LZMA_Stream
+   {
+   public:
+      explicit LZMA_Compression_Stream(size_t level)
+         {
+         if(level == 0)
+            level = 6; // default
+         else if(level > 9)
+            level = 9; // clamp to maximum allowed value
+
+         lzma_ret rc = ::lzma_easy_encoder(streamp(), level, LZMA_CHECK_CRC64);
+
+         if(rc != LZMA_OK)
+            throw Compression_Error("lzam_easy_encoder", ErrorType::LzmaError, rc);
+         }
+   };
+
+class LZMA_Decompression_Stream final : public LZMA_Stream
+   {
+   public:
+      LZMA_Decompression_Stream()
+         {
+         lzma_ret rc = ::lzma_stream_decoder(streamp(), UINT64_MAX,
+                                             LZMA_TELL_UNSUPPORTED_CHECK);
+
+         if(rc != LZMA_OK)
+            throw Compression_Error("lzma_stream_decoder", ErrorType::LzmaError, rc);
+         }
+   };
+
+}
+
+Compression_Stream* LZMA_Compression::make_stream(size_t level) const
+   {
+   return new LZMA_Compression_Stream(level);
+   }
+
+Compression_Stream* LZMA_Decompression::make_stream() const
+   {
+   return new LZMA_Decompression_Stream;
+   }
+
+}
diff --git a/comm/third_party/botan/src/lib/compression/lzma/lzma.h b/comm/third_party/botan/src/lib/compression/lzma/lzma.h
new file mode 100644
index 0000000000..02a1f8c8da
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/lzma/lzma.h
@@ -0,0 +1,42 @@
+/*
+* Lzma Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007 Jack Lloyd
+*     2012 Vojtech Kral
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_LZMA_H_
+#define BOTAN_LZMA_H_
+
+#include <botan/compression.h>
+
+namespace Botan {
+
+/**
+* LZMA Compression
+*/
+class BOTAN_PUBLIC_API(2,0) LZMA_Compression final : public Stream_Compression
+   {
+   public:
+      std::string name() const override { return "LZMA_Compression"; }
+
+   private:
+      Compression_Stream* make_stream(size_t level) const override;
+   };
+
+/**
+* LZMA Deccompression
+*/
+class BOTAN_PUBLIC_API(2,0) LZMA_Decompression final : public Stream_Decompression
+   {
+   public:
+      std::string name() const override { return "LZMA_Decompression"; }
+   private:
+      Compression_Stream* make_stream() const override;
+   };
+
+}
+
+#endif
diff --git a/comm/third_party/botan/src/lib/compression/zlib/info.txt b/comm/third_party/botan/src/lib/compression/zlib/info.txt
new file mode 100644
index 0000000000..1102bc5e1e
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/zlib/info.txt
@@ -0,0 +1,10 @@
+<defines>
+ZLIB -> 20160412
+</defines>
+
+load_on vendor
+
+<libs>
+all!windows -> z
+windows -> zlib
+</libs>
diff --git a/comm/third_party/botan/src/lib/compression/zlib/zlib.cpp b/comm/third_party/botan/src/lib/compression/zlib/zlib.cpp
new file mode 100644
index 0000000000..285bc4e916
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/zlib/zlib.cpp
@@ -0,0 +1,173 @@
+/*
+* Zlib Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007,2014 Jack Lloyd
+*     2006 Matt Johnston
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#include <botan/zlib.h>
+#include <botan/internal/compress_utils.h>
+#include <botan/exceptn.h>
+#include <zlib.h>
+
+namespace Botan {
+
+namespace {
+
+class Zlib_Stream : public Zlib_Style_Stream<z_stream, Bytef>
+   {
+   public:
+      Zlib_Stream()
+         {
+         streamp()->opaque = alloc();
+         streamp()->zalloc = Compression_Alloc_Info::malloc<unsigned int>;
+         streamp()->zfree = Compression_Alloc_Info::free;
+         }
+
+      uint32_t run_flag() const override { return Z_NO_FLUSH; }
+      uint32_t flush_flag() const override { return Z_SYNC_FLUSH; }
+      uint32_t finish_flag() const override { return Z_FINISH; }
+
+      int compute_window_bits(int wbits, int wbits_offset) const
+         {
+         if(wbits_offset == -1)
+            return -wbits;
+         else
+            return wbits + wbits_offset;
+         }
+   };
+
+class Zlib_Compression_Stream : public Zlib_Stream
+   {
+   public:
+      Zlib_Compression_Stream(size_t level, int wbits, int wbits_offset = 0)
+         {
+         wbits = compute_window_bits(wbits, wbits_offset);
+
+         if(level >= 9)
+            level = 9;
+         else if(level == 0)
+            level = 6;
+
+         int rc = ::deflateInit2(streamp(), level, Z_DEFLATED, wbits, 8, Z_DEFAULT_STRATEGY);
+
+         if(rc != Z_OK)
+            throw Compression_Error("deflateInit2", ErrorType::ZlibError, rc);
+         }
+
+      ~Zlib_Compression_Stream()
+         {
+         ::deflateEnd(streamp());
+         }
+
+      bool run(uint32_t flags) override
+         {
+         int rc = ::deflate(streamp(), flags);
+
+         if(rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR)
+            throw Compression_Error("zlib deflate", ErrorType::ZlibError, rc);
+
+         return (rc == Z_STREAM_END);
+         }
+   };
+
+class Zlib_Decompression_Stream : public Zlib_Stream
+   {
+   public:
+      Zlib_Decompression_Stream(int wbits, int wbits_offset = 0)
+         {
+         int rc = ::inflateInit2(streamp(), compute_window_bits(wbits, wbits_offset));
+
+         if(rc != Z_OK)
+            throw Compression_Error("inflateInit2", ErrorType::ZlibError, rc);
+         }
+
+      ~Zlib_Decompression_Stream()
+         {
+         ::inflateEnd(streamp());
+         }
+
+      bool run(uint32_t flags) override
+         {
+         int rc = ::inflate(streamp(), flags);
+
+         if(rc != Z_OK && rc != Z_STREAM_END && rc != Z_BUF_ERROR)
+            throw Compression_Error("zlib inflate", ErrorType::ZlibError, rc);
+
+         return (rc == Z_STREAM_END);
+         }
+   };
+
+class Deflate_Compression_Stream final : public Zlib_Compression_Stream
+   {
+   public:
+      Deflate_Compression_Stream(size_t level, int wbits) :
+         Zlib_Compression_Stream(level, wbits, -1) {}
+   };
+
+class Deflate_Decompression_Stream final : public Zlib_Decompression_Stream
+   {
+   public:
+      explicit Deflate_Decompression_Stream(int wbits) : Zlib_Decompression_Stream(wbits, -1) {}
+   };
+
+class Gzip_Compression_Stream final : public Zlib_Compression_Stream
+   {
+   public:
+      Gzip_Compression_Stream(size_t level, int wbits, uint8_t os_code, uint64_t hdr_time) :
+         Zlib_Compression_Stream(level, wbits, 16)
+         {
+         clear_mem(&m_header, 1);
+         m_header.os = os_code;
+         m_header.time = static_cast<uLong>(hdr_time);
+
+         int rc = deflateSetHeader(streamp(), &m_header);
+         if(rc != Z_OK)
+            throw Compression_Error("deflateSetHeader", ErrorType::ZlibError, rc);
+         }
+
+   private:
+      ::gz_header m_header;
+   };
+
+class Gzip_Decompression_Stream final : public Zlib_Decompression_Stream
+   {
+   public:
+      explicit Gzip_Decompression_Stream(int wbits) : Zlib_Decompression_Stream(wbits, 16) {}
+   };
+
+}
+
+Compression_Stream* Zlib_Compression::make_stream(size_t level) const
+   {
+   return new Zlib_Compression_Stream(level, 15);
+   }
+
+Compression_Stream* Zlib_Decompression::make_stream() const
+   {
+   return new Zlib_Decompression_Stream(15);
+   }
+
+Compression_Stream* Deflate_Compression::make_stream(size_t level) const
+   {
+   return new Deflate_Compression_Stream(level, 15);
+   }
+
+Compression_Stream* Deflate_Decompression::make_stream() const
+   {
+   return new Deflate_Decompression_Stream(15);
+   }
+
+Compression_Stream* Gzip_Compression::make_stream(size_t level) const
+   {
+   return new Gzip_Compression_Stream(level, 15, m_os_code, m_hdr_time);
+   }
+
+Compression_Stream* Gzip_Decompression::make_stream() const
+   {
+   return new Gzip_Decompression_Stream(15);
+   }
+
+}
diff --git a/comm/third_party/botan/src/lib/compression/zlib/zlib.h b/comm/third_party/botan/src/lib/compression/zlib/zlib.h
new file mode 100644
index 0000000000..cc00603446
--- /dev/null
+++ b/comm/third_party/botan/src/lib/compression/zlib/zlib.h
@@ -0,0 +1,89 @@
+/*
+* Zlib Compressor
+* (C) 2001 Peter J Jones
+*     2001-2007,2014 Jack Lloyd
+*
+* Botan is released under the Simplified BSD License (see license.txt)
+*/
+
+#ifndef BOTAN_ZLIB_H_
+#define BOTAN_ZLIB_H_
+
+#include <botan/compression.h>
+
+namespace Botan {
+
+/**
+* Zlib Compression
+*/
+class BOTAN_PUBLIC_API(2,0) Zlib_Compression final : public Stream_Compression
+   {
+   public:
+      std::string name() const override { return "Zlib_Compression"; }
+   private:
+      Compression_Stream* make_stream(size_t level) const override;
+   };
+
+/**
+* Zlib Decompression
+*/
+class BOTAN_PUBLIC_API(2,0) Zlib_Decompression final : public Stream_Decompression
+   {
+   public:
+      std::string name() const override { return "Zlib_Decompression"; }
+   private:
+      Compression_Stream* make_stream() const override;
+   };
+
+/**
+* Deflate Compression
+*/
+class BOTAN_PUBLIC_API(2,0) Deflate_Compression final : public Stream_Compression
+   {
+   public:
+      std::string name() const override { return "Deflate_Compression"; }
+   private:
+      Compression_Stream* make_stream(size_t level) const override;
+   };
+
+/**
+* Deflate Decompression
+*/
+class BOTAN_PUBLIC_API(2,0) Deflate_Decompression final : public Stream_Decompression
+   {
+   public:
+      std::string name() const override { return "Deflate_Decompression"; }
+   private:
+      Compression_Stream* make_stream() const override;
+   };
+
+/**
+* Gzip Compression
+*/
+class BOTAN_PUBLIC_API(2,0) Gzip_Compression final : public Stream_Compression
+   {
+   public:
+      explicit Gzip_Compression(uint8_t os_code = 255, uint64_t hdr_time = 0) :
+         m_hdr_time(hdr_time), m_os_code(os_code) {}
+
+      std::string name() const override { return "Gzip_Compression"; }
+   private:
+      Compression_Stream* make_stream(size_t level) const override;
+      const uint64_t m_hdr_time;
+      const uint8_t m_os_code;
+   };
+
+/**
+* Gzip Decompression
+*/
+class BOTAN_PUBLIC_API(2,0) Gzip_Decompression final : public Stream_Decompression
+   {
+   public:
+      std::string name() const override { return "Gzip_Decompression"; }
+   private:
+      Compression_Stream* make_stream() const override;
+   };
+
+}
+
+#endif