Adding upstream version 1:2.3.19.1+dfsg1.upstream/1%2.3.19.1+dfsg1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 249 insertions, 0 deletions

diff --git a/src/lib-compression/ostream-lz4.c b/src/lib-compression/ostream-lz4.c
new file mode 100644
index 0000000..360f475
--- /dev/null
+++ b/src/lib-compression/ostream-lz4.c
@@ -0,0 +1,249 @@
+/* Copyright (c) 2013-2018 Dovecot authors, see the included COPYING file */
+
+#include "lib.h"
+
+#ifdef HAVE_LZ4
+
+#include "ostream-private.h"
+#include "ostream-zlib.h"
+#include "iostream-lz4.h"
+#include <lz4.h>
+
+#define CHUNK_SIZE OSTREAM_LZ4_CHUNK_SIZE
+
+struct lz4_ostream {
+	struct ostream_private ostream;
+
+	unsigned char compressbuf[CHUNK_SIZE];
+	unsigned int compressbuf_offset;
+
+	/* chunk size, followed by compressed data */
+	unsigned char outbuf[IOSTREAM_LZ4_CHUNK_PREFIX_LEN +
+	                     LZ4_COMPRESSBOUND(CHUNK_SIZE)];
+	unsigned int outbuf_offset, outbuf_used;
+};
+
+/* There is no actual compression level in LZ4, so for legacy
+   reasons we allow 1-9 to avoid breaking anyone's config. */
+int compression_get_min_level_lz4(void)
+{
+	return 1;
+}
+
+int compression_get_default_level_lz4(void)
+{
+	return 1;
+}
+
+int compression_get_max_level_lz4(void)
+{
+	return 9;
+}
+
+static void o_stream_lz4_close(struct iostream_private *stream,
+			       bool close_parent)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+
+	if (close_parent)
+		o_stream_close(zstream->ostream.parent);
+}
+
+static int o_stream_lz4_send_outbuf(struct lz4_ostream *zstream)
+{
+	ssize_t ret;
+	size_t size;
+
+	if (zstream->outbuf_used == 0)
+		return 1;
+
+	size = zstream->outbuf_used - zstream->outbuf_offset;
+	i_assert(size > 0);
+	ret = o_stream_send(zstream->ostream.parent,
+			    zstream->outbuf + zstream->outbuf_offset, size);
+	if (ret < 0) {
+		o_stream_copy_error_from_parent(&zstream->ostream);
+		return -1;
+	}
+	if ((size_t)ret != size) {
+		zstream->outbuf_offset += ret;
+		return 0;
+	}
+	zstream->outbuf_offset = 0;
+	zstream->outbuf_used = 0;
+	return 1;
+}
+
+static int o_stream_lz4_compress(struct lz4_ostream *zstream)
+{
+	uint32_t chunk_size;
+	int ret;
+
+	if (zstream->compressbuf_offset == 0)
+		return 1;
+	if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0)
+		return ret;
+
+	i_assert(zstream->outbuf_offset == 0);
+	i_assert(zstream->outbuf_used == 0);
+
+#if defined(HAVE_LZ4_COMPRESS_DEFAULT)
+	int max_dest_size = LZ4_compressBound(zstream->compressbuf_offset);
+	i_assert(max_dest_size >= 0);
+	if (max_dest_size == 0) {
+		io_stream_set_error(&zstream->ostream.iostream,
+			"lz4-compress: input size %u too large (> %u)",
+			zstream->compressbuf_offset, LZ4_MAX_INPUT_SIZE);
+		zstream->ostream.ostream.stream_errno = EINVAL;
+		return -1;
+	}
+	ret = LZ4_compress_default((void *)zstream->compressbuf,
+				   (void *)(zstream->outbuf +
+				            IOSTREAM_LZ4_CHUNK_PREFIX_LEN),
+				   zstream->compressbuf_offset,
+				   max_dest_size);
+#else
+	ret = LZ4_compress((void *)zstream->compressbuf,
+			   (void *)(zstream->outbuf +
+			            IOSTREAM_LZ4_CHUNK_PREFIX_LEN),
+			   zstream->compressbuf_offset);
+#endif /* defined(HAVE_LZ4_COMPRESS_DEFAULT) */
+	i_assert(ret > 0 && (unsigned int)ret <= sizeof(zstream->outbuf) -
+	         IOSTREAM_LZ4_CHUNK_PREFIX_LEN);
+	zstream->outbuf_used = IOSTREAM_LZ4_CHUNK_PREFIX_LEN + ret;
+	chunk_size = zstream->outbuf_used - IOSTREAM_LZ4_CHUNK_PREFIX_LEN;
+	zstream->outbuf[0] = (chunk_size & 0xff000000) >> 24;
+	zstream->outbuf[1] = (chunk_size & 0x00ff0000) >> 16;
+	zstream->outbuf[2] = (chunk_size & 0x0000ff00) >> 8;
+	zstream->outbuf[3] = (chunk_size & 0x000000ff);
+	zstream->compressbuf_offset = 0;
+	return 1;
+}
+
+static ssize_t
+o_stream_lz4_send_chunk(struct lz4_ostream *zstream,
+			const void *data, size_t size)
+{
+	size_t max_size;
+	ssize_t added_bytes = 0;
+	int ret;
+
+	i_assert(zstream->outbuf_used == 0);
+
+	do {
+		max_size = I_MIN(size, sizeof(zstream->compressbuf) -
+				 zstream->compressbuf_offset);
+		memcpy(zstream->compressbuf + zstream->compressbuf_offset,
+		       data, max_size);
+		zstream->compressbuf_offset += max_size;
+
+		data = CONST_PTR_OFFSET(data, max_size);
+		size -= max_size;
+		added_bytes += max_size;
+
+		if (zstream->compressbuf_offset == sizeof(zstream->compressbuf)) {
+			ret = o_stream_lz4_compress(zstream);
+			if (ret <= 0)
+				return added_bytes != 0 ? added_bytes : ret;
+		}
+	} while (size > 0);
+
+	return added_bytes;
+}
+
+static int o_stream_lz4_flush(struct ostream_private *stream)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+
+	if (o_stream_lz4_compress(zstream) < 0)
+		return -1;
+	if (o_stream_lz4_send_outbuf(zstream) < 0)
+		return -1;
+
+	return o_stream_flush_parent(stream);
+}
+
+static size_t
+o_stream_lz4_get_buffer_used_size(const struct ostream_private *stream)
+{
+	const struct lz4_ostream *zstream =
+		(const struct lz4_ostream *)stream;
+
+	/* outbuf has already compressed data that we're trying to send to the
+	   parent stream. compressbuf isn't included in the return value,
+	   because it needs to be filled up or flushed. */
+	return (zstream->outbuf_used - zstream->outbuf_offset) +
+		o_stream_get_buffer_used_size(stream->parent);
+}
+
+static size_t
+o_stream_lz4_get_buffer_avail_size(const struct ostream_private *stream)
+{
+	const struct lz4_ostream *zstream =
+		(const struct lz4_ostream *)stream;
+
+	/* We're only guaranteed to accept data to compressbuf. The parent
+	   stream might have space, but since compressed data gets written
+	   there it's not really known how much we can actually write there. */
+	return sizeof(zstream->compressbuf) - zstream->compressbuf_offset;
+}
+
+static ssize_t
+o_stream_lz4_sendv(struct ostream_private *stream,
+		    const struct const_iovec *iov, unsigned int iov_count)
+{
+	struct lz4_ostream *zstream = (struct lz4_ostream *)stream;
+	ssize_t ret, bytes = 0;
+	unsigned int i;
+
+	if ((ret = o_stream_lz4_send_outbuf(zstream)) <= 0) {
+		/* error / we still couldn't flush existing data to
+		   parent stream. */
+		return ret;
+	}
+
+	for (i = 0; i < iov_count; i++) {
+		ret = o_stream_lz4_send_chunk(zstream, iov[i].iov_base,
+					      iov[i].iov_len);
+		if (ret < 0)
+			return -1;
+		bytes += ret;
+		if ((size_t)ret != iov[i].iov_len)
+			break;
+	}
+	stream->ostream.offset += bytes;
+	return bytes;
+}
+
+struct ostream *o_stream_create_lz4(struct ostream *output, int level)
+{
+	struct iostream_lz4_header *hdr;
+	struct lz4_ostream *zstream;
+
+	i_assert(level >= 1 && level <= 9);
+
+	zstream = i_new(struct lz4_ostream, 1);
+	zstream->ostream.sendv = o_stream_lz4_sendv;
+	zstream->ostream.flush = o_stream_lz4_flush;
+	zstream->ostream.get_buffer_used_size =
+		o_stream_lz4_get_buffer_used_size;
+	zstream->ostream.get_buffer_avail_size =
+		o_stream_lz4_get_buffer_avail_size;
+	zstream->ostream.iostream.close = o_stream_lz4_close;
+
+	i_assert(sizeof(zstream->outbuf) >= sizeof(*hdr));
+	hdr = (void *)zstream->outbuf;
+	memcpy(hdr->magic, IOSTREAM_LZ4_MAGIC, sizeof(hdr->magic));
+	hdr->max_uncompressed_chunk_size[0] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0xff000000) >> 24;
+	hdr->max_uncompressed_chunk_size[1] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x00ff0000) >> 16;
+	hdr->max_uncompressed_chunk_size[2] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x0000ff00) >> 8;
+	hdr->max_uncompressed_chunk_size[3] =
+		(OSTREAM_LZ4_CHUNK_SIZE & 0x000000ff);
+	zstream->outbuf_used = sizeof(*hdr);
+	return o_stream_create(&zstream->ostream, output,
+			       o_stream_get_fd(output));
+}
+#endif