Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

33 files changed, 21349 insertions, 0 deletions

diff --git a/src/libmime/CMakeLists.txt b/src/libmime/CMakeLists.txt
new file mode 100644
index 0000000..09e5dbf
--- /dev/null
+++ b/src/libmime/CMakeLists.txt
@@ -0,0 +1,19 @@
+# Librspamd mime
+SET(LIBRSPAMDMIMESRC
+		${CMAKE_CURRENT_SOURCE_DIR}/received.cxx
+				${CMAKE_CURRENT_SOURCE_DIR}/email_addr.c
+				${CMAKE_CURRENT_SOURCE_DIR}/mime_expressions.c
+        ${CMAKE_CURRENT_SOURCE_DIR}/scan_result.c
+				${CMAKE_CURRENT_SOURCE_DIR}/images.c
+				${CMAKE_CURRENT_SOURCE_DIR}/message.c
+				${CMAKE_CURRENT_SOURCE_DIR}/archives.c
+				${CMAKE_CURRENT_SOURCE_DIR}/content_type.c
+				${CMAKE_CURRENT_SOURCE_DIR}/mime_headers.c
+				${CMAKE_CURRENT_SOURCE_DIR}/mime_parser.c
+				${CMAKE_CURRENT_SOURCE_DIR}/mime_encoding.c
+				${CMAKE_CURRENT_SOURCE_DIR}/lang_detection.c
+		${CMAKE_CURRENT_SOURCE_DIR}/lang_detection_fasttext.cxx
+		${CMAKE_CURRENT_SOURCE_DIR}/mime_string.cxx
+		)
+
+SET(RSPAMD_MIME ${LIBRSPAMDMIMESRC} PARENT_SCOPE)
\ No newline at end of file
diff --git a/src/libmime/archives.c b/src/libmime/archives.c
new file mode 100644
index 0000000..ea0ea55
--- /dev/null
+++ b/src/libmime/archives.c
@@ -0,0 +1,2057 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "message.h"
+#include "task.h"
+#include "archives.h"
+#include "libmime/mime_encoding.h"
+#include <unicode/uchar.h>
+#include <unicode/utf8.h>
+#include <unicode/utf16.h>
+#include <unicode/ucnv.h>
+
+#define msg_debug_archive(...) rspamd_conditional_debug_fast(NULL, NULL,                                                 \
+															 rspamd_archive_log_id, "archive", task->task_pool->tag.uid, \
+															 G_STRFUNC,                                                  \
+															 __VA_ARGS__)
+
+INIT_LOG_MODULE(archive)
+
+static void
+rspamd_archive_dtor(gpointer p)
+{
+	struct rspamd_archive *arch = p;
+	struct rspamd_archive_file *f;
+	guint i;
+
+	for (i = 0; i < arch->files->len; i++) {
+		f = g_ptr_array_index(arch->files, i);
+
+		if (f->fname) {
+			g_string_free(f->fname, TRUE);
+		}
+
+		g_free(f);
+	}
+
+	g_ptr_array_free(arch->files, TRUE);
+}
+
+static bool
+rspamd_archive_file_try_utf(struct rspamd_task *task,
+							struct rspamd_archive *arch,
+							struct rspamd_archive_file *fentry,
+							const gchar *in, gsize inlen)
+{
+	const gchar *charset = NULL, *p, *end;
+	GString *res;
+
+	charset = rspamd_mime_charset_find_by_content(in, inlen, TRUE);
+
+	if (charset) {
+		UChar *tmp;
+		UErrorCode uc_err = U_ZERO_ERROR;
+		gint32 r, clen, dlen;
+		struct rspamd_charset_converter *conv;
+		UConverter *utf8_converter;
+
+		conv = rspamd_mime_get_converter_cached(charset, task->task_pool,
+												TRUE, &uc_err);
+		utf8_converter = rspamd_get_utf8_converter();
+
+		if (conv == NULL) {
+			msg_info_task("cannot open converter for %s: %s",
+						  charset, u_errorName(uc_err));
+			fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+			fentry->fname = g_string_new_len(in, inlen);
+
+			return false;
+		}
+
+		tmp = g_malloc(sizeof(*tmp) * (inlen + 1));
+		r = rspamd_converter_to_uchars(conv, tmp, inlen + 1,
+									   in, inlen, &uc_err);
+		if (!U_SUCCESS(uc_err)) {
+			msg_info_task("cannot convert data to unicode from %s: %s",
+						  charset, u_errorName(uc_err));
+			g_free(tmp);
+
+			fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+			fentry->fname = g_string_new_len(in, inlen);
+
+			return NULL;
+		}
+
+		int i = 0;
+
+		while (i < r) {
+			UChar32 uc;
+
+			U16_NEXT(tmp, i, r, uc);
+
+			if (IS_ZERO_WIDTH_SPACE(uc) || u_iscntrl(uc)) {
+				msg_info_task("control character in archive file name found: 0x%02xd "
+							  "(filename=%T)",
+							  uc, arch->archive_name);
+				fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+				break;
+			}
+		}
+
+		clen = ucnv_getMaxCharSize(utf8_converter);
+		dlen = UCNV_GET_MAX_BYTES_FOR_STRING(r, clen);
+		res = g_string_sized_new(dlen);
+		r = ucnv_fromUChars(utf8_converter, res->str, dlen, tmp, r, &uc_err);
+
+		if (!U_SUCCESS(uc_err)) {
+			msg_info_task("cannot convert data from unicode from %s: %s",
+						  charset, u_errorName(uc_err));
+			g_free(tmp);
+			g_string_free(res, TRUE);
+			fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+			fentry->fname = g_string_new_len(in, inlen);
+
+			return NULL;
+		}
+
+		g_free(tmp);
+		res->len = r;
+
+		msg_debug_archive("converted from %s to UTF-8 inlen: %z, outlen: %d",
+						  charset, inlen, r);
+		fentry->fname = res;
+	}
+	else {
+		/* Convert unsafe characters to '?' */
+		res = g_string_sized_new(inlen);
+		p = in;
+		end = in + inlen;
+
+		while (p < end) {
+			if (g_ascii_isgraph(*p)) {
+				g_string_append_c(res, *p);
+			}
+			else {
+				g_string_append_c(res, '?');
+
+				if (*p < 0x7f && (g_ascii_iscntrl(*p) || *p == '\0')) {
+					if (!(fentry->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED)) {
+						msg_info_task("suspicious character in archive file name found: 0x%02xd "
+									  "(filename=%T)",
+									  (int) *p, arch->archive_name);
+						fentry->flags |= RSPAMD_ARCHIVE_FILE_OBFUSCATED;
+					}
+				}
+			}
+
+			p++;
+		}
+		fentry->fname = res;
+	}
+
+	return true;
+}
+
+static void
+rspamd_archive_process_zip(struct rspamd_task *task,
+						   struct rspamd_mime_part *part)
+{
+	const guchar *p, *start, *end, *eocd = NULL, *cd;
+	const guint32 eocd_magic = 0x06054b50, cd_basic_len = 46;
+	const guchar cd_magic[] = {0x50, 0x4b, 0x01, 0x02};
+	const guint max_processed = 1024;
+	guint32 cd_offset, cd_size, comp_size, uncomp_size, processed = 0;
+	guint16 extra_len, fname_len, comment_len;
+	struct rspamd_archive *arch;
+	struct rspamd_archive_file *f = NULL;
+
+	/* Zip files have interesting data at the end of archive */
+	p = part->parsed_data.begin + part->parsed_data.len - 1;
+	start = part->parsed_data.begin;
+	end = p;
+
+	/* Search for EOCD:
+	 * 22 bytes is a typical size of eocd without a comment and
+	 * end points one byte after the last character
+	 */
+	p -= 21;
+
+	while (p > start + sizeof(guint32)) {
+		guint32 t;
+
+		if (processed > max_processed) {
+			break;
+		}
+
+		/* XXX: not an efficient approach */
+		memcpy(&t, p, sizeof(t));
+
+		if (GUINT32_FROM_LE(t) == eocd_magic) {
+			eocd = p;
+			break;
+		}
+
+		p--;
+		processed++;
+	}
+
+
+	if (eocd == NULL) {
+		/* Not a zip file */
+		msg_info_task("zip archive is invalid (no EOCD)");
+
+		return;
+	}
+
+	if (end - eocd < 21) {
+		msg_info_task("zip archive is invalid (short EOCD)");
+
+		return;
+	}
+
+
+	memcpy(&cd_size, eocd + 12, sizeof(cd_size));
+	cd_size = GUINT32_FROM_LE(cd_size);
+	memcpy(&cd_offset, eocd + 16, sizeof(cd_offset));
+	cd_offset = GUINT32_FROM_LE(cd_offset);
+
+	/* We need to check sanity as well */
+	if (cd_offset + cd_size > (guint) (eocd - start)) {
+		msg_info_task("zip archive is invalid (bad size/offset for CD)");
+
+		return;
+	}
+
+	cd = start + cd_offset;
+
+	arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+	arch->files = g_ptr_array_new();
+	arch->type = RSPAMD_ARCHIVE_ZIP;
+	if (part->cd) {
+		arch->archive_name = &part->cd->filename;
+	}
+	rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+								  arch);
+
+	while (cd < start + cd_offset + cd_size) {
+		guint16 flags;
+
+		/* Read central directory record */
+		if (eocd - cd < cd_basic_len ||
+			memcmp(cd, cd_magic, sizeof(cd_magic)) != 0) {
+			msg_info_task("zip archive is invalid (bad cd record)");
+
+			return;
+		}
+
+		memcpy(&flags, cd + 8, sizeof(guint16));
+		flags = GUINT16_FROM_LE(flags);
+		memcpy(&comp_size, cd + 20, sizeof(guint32));
+		comp_size = GUINT32_FROM_LE(comp_size);
+		memcpy(&uncomp_size, cd + 24, sizeof(guint32));
+		uncomp_size = GUINT32_FROM_LE(uncomp_size);
+		memcpy(&fname_len, cd + 28, sizeof(fname_len));
+		fname_len = GUINT16_FROM_LE(fname_len);
+		memcpy(&extra_len, cd + 30, sizeof(extra_len));
+		extra_len = GUINT16_FROM_LE(extra_len);
+		memcpy(&comment_len, cd + 32, sizeof(comment_len));
+		comment_len = GUINT16_FROM_LE(comment_len);
+
+		if (cd + fname_len + comment_len + extra_len + cd_basic_len > eocd) {
+			msg_info_task("zip archive is invalid (too large cd record)");
+
+			return;
+		}
+
+		f = g_malloc0(sizeof(*f));
+		rspamd_archive_file_try_utf(task, arch, f, cd + cd_basic_len, fname_len);
+
+		f->compressed_size = comp_size;
+		f->uncompressed_size = uncomp_size;
+
+		if (flags & 0x41u) {
+			f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
+		}
+
+		if (f->fname) {
+			if (f->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED) {
+				arch->flags |= RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES;
+			}
+
+			g_ptr_array_add(arch->files, f);
+			msg_debug_archive("found file in zip archive: %v", f->fname);
+		}
+		else {
+			g_free(f);
+
+			return;
+		}
+
+		/* Process extra fields */
+		const guchar *extra = cd + fname_len + cd_basic_len;
+		p = extra;
+
+		while (p + sizeof(guint16) * 2 < extra + extra_len) {
+			guint16 hid, hlen;
+
+			memcpy(&hid, p, sizeof(guint16));
+			hid = GUINT16_FROM_LE(hid);
+			memcpy(&hlen, p + sizeof(guint16), sizeof(guint16));
+			hlen = GUINT16_FROM_LE(hlen);
+
+			if (hid == 0x0017) {
+				f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
+			}
+
+			p += hlen + sizeof(guint16) * 2;
+		}
+
+		cd += fname_len + comment_len + extra_len + cd_basic_len;
+	}
+
+	part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+	part->specific.arch = arch;
+
+	arch->size = part->parsed_data.len;
+}
+
+static inline gint
+rspamd_archive_rar_read_vint(const guchar *start, gsize remain, guint64 *res)
+{
+	/*
+	 * From http://www.rarlab.com/technote.htm:
+	 * Variable length integer. Can include one or more bytes, where
+	 * lower 7 bits of every byte contain integer data and highest bit
+	 * in every byte is the continuation flag.
+	 * If highest bit is 0, this is the last byte in sequence.
+	 * So first byte contains 7 least significant bits of integer and
+	 * continuation flag. Second byte, if present, contains next 7 bits and so on.
+	 */
+	guint64 t = 0;
+	guint shift = 0;
+	const guchar *p = start;
+
+	while (remain > 0 && shift <= 57) {
+		if (*p & 0x80) {
+			t |= ((guint64) (*p & 0x7f)) << shift;
+		}
+		else {
+			t |= ((guint64) (*p & 0x7f)) << shift;
+			p++;
+			break;
+		}
+
+		shift += 7;
+		p++;
+		remain--;
+	}
+
+	if (remain == 0 || shift > 64) {
+		return -1;
+	}
+
+	*res = GUINT64_FROM_LE(t);
+
+	return p - start;
+}
+
+#define RAR_SKIP_BYTES(n)                                                 \
+	do {                                                                  \
+		if ((n) <= 0) {                                                   \
+			msg_debug_archive("rar archive is invalid (bad skip value)"); \
+			return;                                                       \
+		}                                                                 \
+		if ((gsize) (end - p) < (n)) {                                    \
+			msg_debug_archive("rar archive is invalid (truncated)");      \
+			return;                                                       \
+		}                                                                 \
+		p += (n);                                                         \
+	} while (0)
+
+#define RAR_READ_VINT()                                                    \
+	do {                                                                   \
+		r = rspamd_archive_rar_read_vint(p, end - p, &vint);               \
+		if (r == -1) {                                                     \
+			msg_debug_archive("rar archive is invalid (bad vint)");        \
+			return;                                                        \
+		}                                                                  \
+		else if (r == 0) {                                                 \
+			msg_debug_archive("rar archive is invalid (BAD vint offset)"); \
+			return;                                                        \
+		}                                                                  \
+	} while (0)
+
+#define RAR_READ_VINT_SKIP()                                        \
+	do {                                                            \
+		r = rspamd_archive_rar_read_vint(p, end - p, &vint);        \
+		if (r == -1) {                                              \
+			msg_debug_archive("rar archive is invalid (bad vint)"); \
+			return;                                                 \
+		}                                                           \
+		p += r;                                                     \
+	} while (0)
+
+#define RAR_READ_UINT16(n)                                           \
+	do {                                                             \
+		if (end - p < (glong) sizeof(guint16)) {                     \
+			msg_debug_archive("rar archive is invalid (bad int16)"); \
+			return;                                                  \
+		}                                                            \
+		n = p[0] + (p[1] << 8);                                      \
+		p += sizeof(guint16);                                        \
+	} while (0)
+
+#define RAR_READ_UINT32(n)                                                                    \
+	do {                                                                                      \
+		if (end - p < (glong) sizeof(guint32)) {                                              \
+			msg_debug_archive("rar archive is invalid (bad int32)");                          \
+			return;                                                                           \
+		}                                                                                     \
+		n = (guint) p[0] + ((guint) p[1] << 8) + ((guint) p[2] << 16) + ((guint) p[3] << 24); \
+		p += sizeof(guint32);                                                                 \
+	} while (0)
+
+static void
+rspamd_archive_process_rar_v4(struct rspamd_task *task, const guchar *start,
+							  const guchar *end, struct rspamd_mime_part *part)
+{
+	const guchar *p = start, *start_section;
+	guint8 type;
+	guint flags;
+	guint64 sz, comp_sz = 0, uncomp_sz = 0;
+	struct rspamd_archive *arch;
+	struct rspamd_archive_file *f;
+
+	arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+	arch->files = g_ptr_array_new();
+	arch->type = RSPAMD_ARCHIVE_RAR;
+	if (part->cd) {
+		arch->archive_name = &part->cd->filename;
+	}
+	rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+								  arch);
+
+	while (p < end) {
+		/* Crc16 */
+		start_section = p;
+		RAR_SKIP_BYTES(sizeof(guint16));
+		type = *p;
+		p++;
+		RAR_READ_UINT16(flags);
+
+		if (type == 0x73) {
+			/* Main header, check for encryption */
+			if (flags & 0x80) {
+				arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+				goto end;
+			}
+		}
+
+		RAR_READ_UINT16(sz);
+
+		if (flags & 0x8000) {
+			/* We also need to read ADD_SIZE element */
+			guint32 tmp;
+
+			RAR_READ_UINT32(tmp);
+			sz += tmp;
+			/* This is also used as PACK_SIZE */
+			comp_sz = tmp;
+		}
+
+		if (sz == 0) {
+			/* Zero sized block - error */
+			msg_debug_archive("rar archive is invalid (zero size block)");
+
+			return;
+		}
+
+		if (type == 0x74) {
+			guint fname_len;
+
+			/* File header */
+			/* Uncompressed size */
+			RAR_READ_UINT32(uncomp_sz);
+			/* Skip to NAME_SIZE element */
+			RAR_SKIP_BYTES(11);
+			RAR_READ_UINT16(fname_len);
+
+			if (fname_len == 0 || fname_len > (gsize) (end - p)) {
+				msg_debug_archive("rar archive is invalid (bad filename size: %d)",
+								  fname_len);
+
+				return;
+			}
+
+			/* Attrs */
+			RAR_SKIP_BYTES(4);
+
+			if (flags & 0x100) {
+				/* We also need to read HIGH_PACK_SIZE */
+				guint32 tmp;
+
+				RAR_READ_UINT32(tmp);
+				sz += tmp;
+				comp_sz += tmp;
+				/* HIGH_UNP_SIZE  */
+				RAR_READ_UINT32(tmp);
+				uncomp_sz += tmp;
+			}
+
+			f = g_malloc0(sizeof(*f));
+
+			if (flags & 0x200) {
+				/* We have unicode + normal version */
+				guchar *tmp;
+
+				tmp = memchr(p, '\0', fname_len);
+
+				if (tmp != NULL) {
+					/* Just use ASCII version */
+					rspamd_archive_file_try_utf(task, arch, f, p, tmp - p);
+					msg_debug_archive("found ascii filename in rarv4 archive: %v",
+									  f->fname);
+				}
+				else {
+					/* We have UTF8 filename, use it as is */
+					rspamd_archive_file_try_utf(task, arch, f, p, fname_len);
+					msg_debug_archive("found utf filename in rarv4 archive: %v",
+									  f->fname);
+				}
+			}
+			else {
+				rspamd_archive_file_try_utf(task, arch, f, p, fname_len);
+				msg_debug_archive("found ascii (old) filename in rarv4 archive: %v",
+								  f->fname);
+			}
+
+			f->compressed_size = comp_sz;
+			f->uncompressed_size = uncomp_sz;
+
+			if (flags & 0x4) {
+				f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
+			}
+
+			if (f->fname) {
+				if (f->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED) {
+					arch->flags |= RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES;
+				}
+				g_ptr_array_add(arch->files, f);
+			}
+			else {
+				g_free(f);
+			}
+		}
+
+		p = start_section;
+		RAR_SKIP_BYTES(sz);
+	}
+
+end:
+	part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+	part->specific.arch = arch;
+	arch->size = part->parsed_data.len;
+}
+
+static void
+rspamd_archive_process_rar(struct rspamd_task *task,
+						   struct rspamd_mime_part *part)
+{
+	const guchar *p, *end, *section_start;
+	const guchar rar_v5_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x01, 0x00},
+				 rar_v4_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07, 0x00};
+	const guint rar_encrypted_header = 4, rar_main_header = 1,
+				rar_file_header = 2;
+	guint64 vint, sz, comp_sz = 0, uncomp_sz = 0, flags = 0, type = 0,
+					  extra_sz = 0;
+	struct rspamd_archive *arch;
+	struct rspamd_archive_file *f;
+	gint r;
+
+	p = part->parsed_data.begin;
+	end = p + part->parsed_data.len;
+
+	if ((gsize) (end - p) <= sizeof(rar_v5_magic)) {
+		msg_debug_archive("rar archive is invalid (too small)");
+
+		return;
+	}
+
+	if (memcmp(p, rar_v5_magic, sizeof(rar_v5_magic)) == 0) {
+		p += sizeof(rar_v5_magic);
+	}
+	else if (memcmp(p, rar_v4_magic, sizeof(rar_v4_magic)) == 0) {
+		p += sizeof(rar_v4_magic);
+
+		rspamd_archive_process_rar_v4(task, p, end, part);
+		return;
+	}
+	else {
+		msg_debug_archive("rar archive is invalid (no rar magic)");
+
+		return;
+	}
+
+	/* Rar v5 format */
+	arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+	arch->files = g_ptr_array_new();
+	arch->type = RSPAMD_ARCHIVE_RAR;
+	if (part->cd) {
+		arch->archive_name = &part->cd->filename;
+	}
+	rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+								  arch);
+
+	/* Now we can have either encryption header or archive header */
+	/* Crc 32 */
+	RAR_SKIP_BYTES(sizeof(guint32));
+	/* Size */
+	RAR_READ_VINT_SKIP();
+	sz = vint;
+	/* Type */
+	section_start = p;
+	RAR_READ_VINT_SKIP();
+	type = vint;
+	/* Header flags */
+	RAR_READ_VINT_SKIP();
+	flags = vint;
+
+	if (flags & 0x1) {
+		/* Have extra zone */
+		RAR_READ_VINT_SKIP();
+	}
+	if (flags & 0x2) {
+		/* Data zone is presented */
+		RAR_READ_VINT_SKIP();
+		sz += vint;
+	}
+
+	if (type == rar_encrypted_header) {
+		/* We can't read any further information as archive is encrypted */
+		arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+		goto end;
+	}
+	else if (type != rar_main_header) {
+		msg_debug_archive("rar archive is invalid (bad main header)");
+
+		return;
+	}
+
+	/* Nothing useful in main header */
+	p = section_start;
+	RAR_SKIP_BYTES(sz);
+
+	while (p < end) {
+		gboolean has_extra = FALSE;
+		/* Read the next header */
+		/* Crc 32 */
+		RAR_SKIP_BYTES(sizeof(guint32));
+		/* Size */
+		RAR_READ_VINT_SKIP();
+
+		sz = vint;
+		if (sz == 0) {
+			/* Zero sized block - error */
+			msg_debug_archive("rar archive is invalid (zero size block)");
+
+			return;
+		}
+
+		section_start = p;
+		/* Type */
+		RAR_READ_VINT_SKIP();
+		type = vint;
+		/* Header flags */
+		RAR_READ_VINT_SKIP();
+		flags = vint;
+
+		if (flags & 0x1) {
+			/* Have extra zone */
+			RAR_READ_VINT_SKIP();
+			extra_sz = vint;
+			has_extra = TRUE;
+		}
+
+		if (flags & 0x2) {
+			/* Data zone is presented */
+			RAR_READ_VINT_SKIP();
+			sz += vint;
+			comp_sz = vint;
+		}
+
+		if (type != rar_file_header) {
+			p = section_start;
+			RAR_SKIP_BYTES(sz);
+		}
+		else {
+			/* We have a file header, go forward */
+			guint64 fname_len;
+			bool is_directory = false;
+
+			/* File header specific flags */
+			RAR_READ_VINT_SKIP();
+			flags = vint;
+
+			/* Unpacked size */
+			RAR_READ_VINT_SKIP();
+			uncomp_sz = vint;
+			/* Attributes */
+			RAR_READ_VINT_SKIP();
+
+			if (flags & 0x2) {
+				/* Unix mtime */
+				RAR_SKIP_BYTES(sizeof(guint32));
+			}
+			if (flags & 0x4) {
+				/* Crc32 */
+				RAR_SKIP_BYTES(sizeof(guint32));
+			}
+			if (flags & 0x1) {
+				/* Ignore directories for sanity purposes */
+				is_directory = true;
+				msg_debug_archive("skip directory record in a rar archive");
+			}
+
+			if (!is_directory) {
+				/* Compression */
+				RAR_READ_VINT_SKIP();
+				/* Host OS */
+				RAR_READ_VINT_SKIP();
+				/* Filename length (finally!) */
+				RAR_READ_VINT_SKIP();
+				fname_len = vint;
+
+				if (fname_len == 0 || fname_len > (gsize) (end - p)) {
+					msg_debug_archive("rar archive is invalid (bad filename size)");
+
+					return;
+				}
+
+				f = g_malloc0(sizeof(*f));
+				f->uncompressed_size = uncomp_sz;
+				f->compressed_size = comp_sz;
+				rspamd_archive_file_try_utf(task, arch, f, p, fname_len);
+
+				if (f->fname) {
+					msg_debug_archive("added rarv5 file: %v", f->fname);
+					g_ptr_array_add(arch->files, f);
+					if (f->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED) {
+						arch->flags |= RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES;
+					}
+				}
+				else {
+					g_free(f);
+					f = NULL;
+				}
+
+				if (f && has_extra && extra_sz > 0 &&
+					p + fname_len + extra_sz < end) {
+					/* Try to find encryption record in extra field */
+					const guchar *ex = p + fname_len;
+
+					while (ex < p + extra_sz) {
+						const guchar *t;
+						gint64 cur_sz = 0, sec_type = 0;
+
+						r = rspamd_archive_rar_read_vint(ex, extra_sz, &cur_sz);
+						if (r == -1) {
+							msg_debug_archive("rar archive is invalid (bad vint)");
+							return;
+						}
+
+						t = ex + r;
+
+						r = rspamd_archive_rar_read_vint(t, extra_sz - r, &sec_type);
+						if (r == -1) {
+							msg_debug_archive("rar archive is invalid (bad vint)");
+							return;
+						}
+
+						if (sec_type == 0x01) {
+							f->flags |= RSPAMD_ARCHIVE_FILE_ENCRYPTED;
+							arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+							break;
+						}
+
+						ex += cur_sz;
+					}
+				}
+			}
+
+			/* Restore p to the beginning of the header */
+			p = section_start;
+			RAR_SKIP_BYTES(sz);
+		}
+	}
+
+end:
+	part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+	part->specific.arch = arch;
+	arch->size = part->parsed_data.len;
+}
+
+static inline gint
+rspamd_archive_7zip_read_vint(const guchar *start, gsize remain, guint64 *res)
+{
+	/*
+	 * REAL_UINT64 means real UINT64.
+	 * UINT64 means real UINT64 encoded with the following scheme:
+	 *
+	 * Size of encoding sequence depends from first byte:
+	 * First_Byte  Extra_Bytes        Value
+	 * (binary)
+	 * 0xxxxxxx               : ( xxxxxxx           )
+	 * 10xxxxxx    BYTE y[1]  : (  xxxxxx << (8 * 1)) + y
+	 * 110xxxxx    BYTE y[2]  : (   xxxxx << (8 * 2)) + y
+	 * ...
+	 * 1111110x    BYTE y[6]  : (       x << (8 * 6)) + y
+	 * 11111110    BYTE y[7]  :                         y
+	 * 11111111    BYTE y[8]  :                         y
+	 */
+	guchar t;
+
+	if (remain == 0) {
+		return -1;
+	}
+
+	t = *start;
+
+	if (!isset(&t, 7)) {
+		/* Trivial case */
+		*res = t;
+		return 1;
+	}
+	else if (t == 0xFF) {
+		if (remain >= sizeof(guint64) + 1) {
+			memcpy(res, start + 1, sizeof(guint64));
+			*res = GUINT64_FROM_LE(*res);
+
+			return sizeof(guint64) + 1;
+		}
+	}
+	else {
+		gint cur_bit = 6, intlen = 1;
+		const guchar bmask = 0xFF;
+		guint64 tgt;
+
+		while (cur_bit > 0) {
+			if (!isset(&t, cur_bit)) {
+				if (remain >= intlen + 1) {
+					memcpy(&tgt, start + 1, intlen);
+					tgt = GUINT64_FROM_LE(tgt);
+					/* Shift back */
+					tgt >>= sizeof(tgt) - NBBY * intlen;
+					/* Add masked value */
+					tgt += (guint64) (t & (bmask >> (NBBY - cur_bit)))
+						   << (NBBY * intlen);
+					*res = tgt;
+
+					return intlen + 1;
+				}
+			}
+			cur_bit--;
+			intlen++;
+		}
+	}
+
+	return -1;
+}
+
+#define SZ_READ_VINT_SKIP()                                        \
+	do {                                                           \
+		r = rspamd_archive_7zip_read_vint(p, end - p, &vint);      \
+		if (r == -1) {                                             \
+			msg_debug_archive("7z archive is invalid (bad vint)"); \
+			return;                                                \
+		}                                                          \
+		p += r;                                                    \
+	} while (0)
+#define SZ_READ_VINT(var)                                                        \
+	do {                                                                         \
+		int r;                                                                   \
+		r = rspamd_archive_7zip_read_vint(p, end - p, &(var));                   \
+		if (r == -1) {                                                           \
+			msg_debug_archive("7z archive is invalid (bad vint): %s", G_STRLOC); \
+			return NULL;                                                         \
+		}                                                                        \
+		p += r;                                                                  \
+	} while (0)
+
+#define SZ_READ_UINT64(n)                                                            \
+	do {                                                                             \
+		if (end - p < (goffset) sizeof(guint64)) {                                   \
+			msg_debug_archive("7zip archive is invalid (bad uint64): %s", G_STRLOC); \
+			return;                                                                  \
+		}                                                                            \
+		memcpy(&(n), p, sizeof(guint64));                                            \
+		n = GUINT64_FROM_LE(n);                                                      \
+		p += sizeof(guint64);                                                        \
+	} while (0)
+#define SZ_SKIP_BYTES(n)                                                                                                                             \
+	do {                                                                                                                                             \
+		if (end - p >= (n)) {                                                                                                                        \
+			p += (n);                                                                                                                                \
+		}                                                                                                                                            \
+		else {                                                                                                                                       \
+			msg_debug_archive("7zip archive is invalid (truncated); wanted to read %d bytes, %d avail: %s", (gint) (n), (gint) (end - p), G_STRLOC); \
+			return NULL;                                                                                                                             \
+		}                                                                                                                                            \
+	} while (0)
+
+enum rspamd_7zip_header_mark {
+	kEnd = 0x00,
+	kHeader = 0x01,
+	kArchiveProperties = 0x02,
+	kAdditionalStreamsInfo = 0x03,
+	kMainStreamsInfo = 0x04,
+	kFilesInfo = 0x05,
+	kPackInfo = 0x06,
+	kUnPackInfo = 0x07,
+	kSubStreamsInfo = 0x08,
+	kSize = 0x09,
+	kCRC = 0x0A,
+	kFolder = 0x0B,
+	kCodersUnPackSize = 0x0C,
+	kNumUnPackStream = 0x0D,
+	kEmptyStream = 0x0E,
+	kEmptyFile = 0x0F,
+	kAnti = 0x10,
+	kName = 0x11,
+	kCTime = 0x12,
+	kATime = 0x13,
+	kMTime = 0x14,
+	kWinAttributes = 0x15,
+	kComment = 0x16,
+	kEncodedHeader = 0x17,
+	kStartPos = 0x18,
+	kDummy = 0x19,
+};
+
+
+#define _7Z_CRYPTO_MAIN_ZIP 0x06F10101        /* Main Zip crypto algo */
+#define _7Z_CRYPTO_RAR_29 0x06F10303          /* Rar29 AES-128 + (modified SHA-1) */
+#define _7Z_CRYPTO_AES_256_SHA_256 0x06F10701 /* AES-256 + SHA-256 */
+
+#define IS_SZ_ENCRYPTED(codec_id) (((codec_id) == _7Z_CRYPTO_MAIN_ZIP) || \
+								   ((codec_id) == _7Z_CRYPTO_RAR_29) ||   \
+								   ((codec_id) == _7Z_CRYPTO_AES_256_SHA_256))
+
+static const guchar *
+rspamd_7zip_read_bits(struct rspamd_task *task,
+					  const guchar *p, const guchar *end,
+					  struct rspamd_archive *arch, guint nbits,
+					  guint *pbits_set)
+{
+	unsigned mask = 0, avail = 0, i;
+	gboolean bit_set = 0;
+
+	for (i = 0; i < nbits; i++) {
+		if (mask == 0) {
+			avail = *p;
+			SZ_SKIP_BYTES(1);
+			mask = 0x80;
+		}
+
+		bit_set = (avail & mask) ? 1 : 0;
+
+		if (bit_set && pbits_set) {
+			(*pbits_set)++;
+		}
+
+		mask >>= 1;
+	}
+
+	return p;
+}
+
+static const guchar *
+rspamd_7zip_read_digest(struct rspamd_task *task,
+						const guchar *p, const guchar *end,
+						struct rspamd_archive *arch,
+						guint64 num_streams,
+						guint *pdigest_read)
+{
+	guchar all_defined = *p;
+	guint64 i;
+	guint num_defined = 0;
+	/*
+	 * BYTE AllAreDefined
+	 *  if (AllAreDefined == 0)
+	 *  {
+	 *    for(NumStreams)
+	 *    BIT Defined
+	 *  }
+	 *  UINT32 CRCs[NumDefined]
+	 */
+	SZ_SKIP_BYTES(1);
+
+	if (all_defined) {
+		num_defined = num_streams;
+	}
+	else {
+		if (num_streams > 8192) {
+			/* Gah */
+			return NULL;
+		}
+
+		p = rspamd_7zip_read_bits(task, p, end, arch, num_streams, &num_defined);
+
+		if (p == NULL) {
+			return NULL;
+		}
+	}
+
+	for (i = 0; i < num_defined; i++) {
+		SZ_SKIP_BYTES(sizeof(guint32));
+	}
+
+	if (pdigest_read) {
+		*pdigest_read = num_defined;
+	}
+
+	return p;
+}
+
+static const guchar *
+rspamd_7zip_read_pack_info(struct rspamd_task *task,
+						   const guchar *p, const guchar *end,
+						   struct rspamd_archive *arch)
+{
+	guint64 pack_pos = 0, pack_streams = 0, i, cur_sz;
+	guint num_digests = 0;
+	guchar t;
+	/*
+	 *  UINT64 PackPos
+	 *  UINT64 NumPackStreams
+	 *
+	 *  []
+	 *  BYTE NID::kSize    (0x09)
+	 *  UINT64 PackSizes[NumPackStreams]
+	 *  []
+	 *
+	 *  []
+	 *  BYTE NID::kCRC      (0x0A)
+	 *  PackStreamDigests[NumPackStreams]
+	 *  []
+	 *  BYTE NID::kEnd
+	 */
+
+	SZ_READ_VINT(pack_pos);
+	SZ_READ_VINT(pack_streams);
+
+	while (p != NULL && p < end) {
+		t = *p;
+		SZ_SKIP_BYTES(1);
+		msg_debug_archive("7zip: read pack info %xc", t);
+
+		switch (t) {
+		case kSize:
+			/* We need to skip pack_streams VINTS */
+			for (i = 0; i < pack_streams; i++) {
+				SZ_READ_VINT(cur_sz);
+			}
+			break;
+		case kCRC:
+			/* CRCs are more complicated */
+			p = rspamd_7zip_read_digest(task, p, end, arch, pack_streams,
+										&num_digests);
+			break;
+		case kEnd:
+			goto end;
+			break;
+		default:
+			p = NULL;
+			msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+			goto end;
+			break;
+		}
+	}
+
+end:
+
+	return p;
+}
+
+static const guchar *
+rspamd_7zip_read_folder(struct rspamd_task *task,
+						const guchar *p, const guchar *end,
+						struct rspamd_archive *arch, guint *pnstreams, guint *ndigests)
+{
+	guint64 ncoders = 0, i, j, noutstreams = 0, ninstreams = 0;
+
+	SZ_READ_VINT(ncoders);
+
+	for (i = 0; i < ncoders && p != NULL && p < end; i++) {
+		guint64 sz, tmp;
+		guchar t;
+		/*
+		 * BYTE
+		 * {
+		 *   0:3 CodecIdSize
+		 *   4:  Is Complex Coder
+		 *   5:  There Are Attributes
+		 *   6:  Reserved
+		 *   7:  There are more alternative methods. (Not used anymore, must be 0).
+		 * }
+		 * BYTE CodecId[CodecIdSize]
+		 * if (Is Complex Coder)
+		 * {
+		 *   UINT64 NumInStreams;
+		 *   UINT64 NumOutStreams;
+		 * }
+		 * if (There Are Attributes)
+		 * {
+		 *   UINT64 PropertiesSize
+		 *   BYTE Properties[PropertiesSize]
+		 * }
+		 */
+		t = *p;
+		SZ_SKIP_BYTES(1);
+		sz = t & 0xF;
+		/* Codec ID */
+		tmp = 0;
+		for (j = 0; j < sz; j++) {
+			tmp <<= 8;
+			tmp += p[j];
+		}
+
+		msg_debug_archive("7zip: read codec id: %L", tmp);
+
+		if (IS_SZ_ENCRYPTED(tmp)) {
+			arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+		}
+
+		SZ_SKIP_BYTES(sz);
+
+		if (t & (1u << 4)) {
+			/* Complex */
+			SZ_READ_VINT(tmp); /* InStreams */
+			ninstreams += tmp;
+			SZ_READ_VINT(tmp); /* OutStreams */
+			noutstreams += tmp;
+		}
+		else {
+			/* XXX: is it correct ? */
+			noutstreams++;
+			ninstreams++;
+		}
+		if (t & (1u << 5)) {
+			/* Attributes ... */
+			SZ_READ_VINT(tmp); /* Size of attrs */
+			SZ_SKIP_BYTES(tmp);
+		}
+	}
+
+	if (noutstreams > 1) {
+		/* BindPairs, WTF, huh */
+		for (i = 0; i < noutstreams - 1; i++) {
+			guint64 tmp;
+
+			SZ_READ_VINT(tmp);
+			SZ_READ_VINT(tmp);
+		}
+	}
+
+	gint64 npacked = (gint64) ninstreams - (gint64) noutstreams + 1;
+	msg_debug_archive("7zip: instreams=%L, outstreams=%L, packed=%L",
+					  ninstreams, noutstreams, npacked);
+
+	if (npacked > 1) {
+		/* Gah... */
+		for (i = 0; i < npacked; i++) {
+			guint64 tmp;
+
+			SZ_READ_VINT(tmp);
+		}
+	}
+
+	*pnstreams = noutstreams;
+	(*ndigests) += npacked;
+
+	return p;
+}
+
+static const guchar *
+rspamd_7zip_read_coders_info(struct rspamd_task *task,
+							 const guchar *p, const guchar *end,
+							 struct rspamd_archive *arch,
+							 guint *pnum_folders, guint *pnum_nodigest)
+{
+	guint64 num_folders = 0, i, tmp;
+	guchar t;
+	guint *folder_nstreams = NULL, num_digests = 0, digests_read = 0;
+
+	while (p != NULL && p < end) {
+		/*
+		 * BYTE NID::kFolder  (0x0B)
+		 *  UINT64 NumFolders
+		 *  BYTE External
+		 *  switch(External)
+		 *  {
+		 * 	case 0:
+		 * 	  Folders[NumFolders]
+		 * 	case 1:
+		 * 	  UINT64 DataStreamIndex
+		 *   }
+		 *   BYTE ID::kCodersUnPackSize  (0x0C)
+		 *   for(Folders)
+		 * 	for(Folder.NumOutStreams)
+		 * 	 UINT64 UnPackSize;
+		 *   []
+		 *   BYTE NID::kCRC   (0x0A)
+		 *   UnPackDigests[NumFolders]
+		 *   []
+		 *   BYTE NID::kEnd
+		 */
+
+		t = *p;
+		SZ_SKIP_BYTES(1);
+		msg_debug_archive("7zip: read coders info %xc", t);
+
+		switch (t) {
+		case kFolder:
+			SZ_READ_VINT(num_folders);
+			msg_debug_archive("7zip: nfolders=%L", num_folders);
+
+			if (*p != 0) {
+				/* External folders */
+				SZ_SKIP_BYTES(1);
+				SZ_READ_VINT(tmp);
+			}
+			else {
+				SZ_SKIP_BYTES(1);
+
+				if (num_folders > 8192) {
+					/* Gah */
+					return NULL;
+				}
+
+				if (folder_nstreams) {
+					g_free(folder_nstreams);
+				}
+
+				folder_nstreams = g_malloc(sizeof(int) * num_folders);
+
+				for (i = 0; i < num_folders && p != NULL && p < end; i++) {
+					p = rspamd_7zip_read_folder(task, p, end, arch,
+												&folder_nstreams[i], &num_digests);
+				}
+			}
+			break;
+		case kCodersUnPackSize:
+			for (i = 0; i < num_folders && p != NULL && p < end; i++) {
+				if (folder_nstreams) {
+					for (guint j = 0; j < folder_nstreams[i]; j++) {
+						SZ_READ_VINT(tmp); /* Unpacked size */
+						msg_debug_archive("7zip: unpacked size "
+										  "(folder=%d, stream=%d) = %L",
+										  (gint) i, j, tmp);
+					}
+				}
+				else {
+					msg_err_task("internal 7zip error");
+				}
+			}
+			break;
+		case kCRC:
+			/*
+			 * Here are dragons. Spec tells that here there could be up
+			 * to nfolders digests. However, according to the actual source
+			 * code, in case of multiple out streams there should be digests
+			 * for all out streams.
+			 *
+			 * In the real life (tm) it is even more idiotic: all these digests
+			 * are in another section! But that section needs number of digests
+			 * that are absent here. It is the most stupid thing I've ever seen
+			 * in any file format.
+			 *
+			 * I hope there *WAS* some reason to do such shit...
+			 */
+			p = rspamd_7zip_read_digest(task, p, end, arch, num_digests,
+										&digests_read);
+			break;
+		case kEnd:
+			goto end;
+			break;
+		default:
+			p = NULL;
+			msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+			goto end;
+			break;
+		}
+	}
+
+end:
+
+	if (pnum_nodigest) {
+		*pnum_nodigest = num_digests - digests_read;
+	}
+	if (pnum_folders) {
+		*pnum_folders = num_folders;
+	}
+
+	if (folder_nstreams) {
+		g_free(folder_nstreams);
+	}
+
+	return p;
+}
+
+static const guchar *
+rspamd_7zip_read_substreams_info(struct rspamd_task *task,
+								 const guchar *p, const guchar *end,
+								 struct rspamd_archive *arch,
+								 guint num_folders, guint num_nodigest)
+{
+	guchar t;
+	guint i;
+	guint64 *folder_nstreams;
+
+	if (num_folders > 8192) {
+		/* Gah */
+		return NULL;
+	}
+
+	folder_nstreams = g_alloca(sizeof(guint64) * num_folders);
+	memset(folder_nstreams, 0, sizeof(guint64) * num_folders);
+
+	while (p != NULL && p < end) {
+		/*
+		 * []
+		 *  BYTE NID::kNumUnPackStream; (0x0D)
+		 *  UINT64 NumUnPackStreamsInFolders[NumFolders];
+		 *  []
+		 *
+		 *  []
+		 *  BYTE NID::kSize  (0x09)
+		 *  UINT64 UnPackSizes[??]
+		 *  []
+		 *
+		 *
+		 *  []
+		 *  BYTE NID::kCRC  (0x0A)
+		 *  Digests[Number of streams with unknown CRC]
+		 *  []
+
+		 */
+		t = *p;
+		SZ_SKIP_BYTES(1);
+
+		msg_debug_archive("7zip: read substream info %xc", t);
+
+		switch (t) {
+		case kNumUnPackStream:
+			for (i = 0; i < num_folders; i++) {
+				guint64 tmp;
+
+				SZ_READ_VINT(tmp);
+				folder_nstreams[i] = tmp;
+			}
+			break;
+		case kCRC:
+			/*
+			 * Read the comment in the rspamd_7zip_read_coders_info
+			 */
+			p = rspamd_7zip_read_digest(task, p, end, arch, num_nodigest,
+										NULL);
+			break;
+		case kSize:
+			/*
+			 * Another brain damaged logic, but we have to support it
+			 * as there are no ways to proceed without it.
+			 * In fact, it is just absent in the real life...
+			 */
+			for (i = 0; i < num_folders; i++) {
+				for (guint j = 0; j < folder_nstreams[i]; j++) {
+					guint64 tmp;
+
+					SZ_READ_VINT(tmp); /* Who cares indeed */
+				}
+			}
+			break;
+		case kEnd:
+			goto end;
+			break;
+		default:
+			p = NULL;
+			msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+			goto end;
+			break;
+		}
+	}
+
+end:
+	return p;
+}
+
+static const guchar *
+rspamd_7zip_read_main_streams_info(struct rspamd_task *task,
+								   const guchar *p, const guchar *end,
+								   struct rspamd_archive *arch)
+{
+	guchar t;
+	guint num_folders = 0, unknown_digests = 0;
+
+	while (p != NULL && p < end) {
+		t = *p;
+		SZ_SKIP_BYTES(1);
+		msg_debug_archive("7zip: read main streams info %xc", t);
+
+		/*
+		 *
+		 *  []
+		 *  PackInfo
+		 *  []
+
+		 *  []
+		 *  CodersInfo
+		 *  []
+		 *
+		 *  []
+		 *  SubStreamsInfo
+		 *  []
+		 *
+		 *  BYTE NID::kEnd
+		 */
+		switch (t) {
+		case kPackInfo:
+			p = rspamd_7zip_read_pack_info(task, p, end, arch);
+			break;
+		case kUnPackInfo:
+			p = rspamd_7zip_read_coders_info(task, p, end, arch, &num_folders,
+											 &unknown_digests);
+			break;
+		case kSubStreamsInfo:
+			p = rspamd_7zip_read_substreams_info(task, p, end, arch, num_folders,
+												 unknown_digests);
+			break;
+			break;
+		case kEnd:
+			goto end;
+			break;
+		default:
+			p = NULL;
+			msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+			goto end;
+			break;
+		}
+	}
+
+end:
+	return p;
+}
+
+static const guchar *
+rspamd_7zip_read_archive_props(struct rspamd_task *task,
+							   const guchar *p, const guchar *end,
+							   struct rspamd_archive *arch)
+{
+	guchar proptype;
+	guint64 proplen;
+
+	/*
+	 * for (;;)
+	 * {
+	 *   BYTE PropertyType;
+	 *   if (aType == 0)
+	 *     break;
+	 *   UINT64 PropertySize;
+	 *   BYTE PropertyData[PropertySize];
+	 * }
+	 */
+
+	if (p != NULL) {
+		proptype = *p;
+		SZ_SKIP_BYTES(1);
+
+		while (proptype != 0) {
+			SZ_READ_VINT(proplen);
+
+			if (p + proplen < end) {
+				p += proplen;
+			}
+			else {
+				return NULL;
+			}
+
+			proptype = *p;
+			SZ_SKIP_BYTES(1);
+		}
+	}
+
+	return p;
+}
+
+static GString *
+rspamd_7zip_ucs2_to_utf8(struct rspamd_task *task, const guchar *p,
+						 const guchar *end)
+{
+	GString *res;
+	goffset dest_pos = 0, src_pos = 0;
+	const gsize len = (end - p) / sizeof(guint16);
+	guint16 *up;
+	UChar32 wc;
+	UBool is_error = 0;
+
+	res = g_string_sized_new((end - p) * 3 / 2 + sizeof(wc) + 1);
+	up = (guint16 *) p;
+
+	while (src_pos < len) {
+		U16_NEXT(up, src_pos, len, wc);
+
+		if (wc > 0) {
+			U8_APPEND(res->str, dest_pos,
+					  res->allocated_len - 1,
+					  wc, is_error);
+		}
+
+		if (is_error) {
+			g_string_free(res, TRUE);
+
+			return NULL;
+		}
+	}
+
+	g_assert(dest_pos < res->allocated_len);
+
+	res->len = dest_pos;
+	res->str[dest_pos] = '\0';
+
+	return res;
+}
+
+static const guchar *
+rspamd_7zip_read_files_info(struct rspamd_task *task,
+							const guchar *p, const guchar *end,
+							struct rspamd_archive *arch)
+{
+	guint64 nfiles = 0, sz, i;
+	guchar t, b;
+	struct rspamd_archive_file *fentry;
+
+	SZ_READ_VINT(nfiles);
+
+	for (; p != NULL && p < end;) {
+		t = *p;
+		SZ_SKIP_BYTES(1);
+
+		msg_debug_archive("7zip: read file data type %xc", t);
+
+		if (t == kEnd) {
+			goto end;
+		}
+
+		/* This is SO SPECIAL, gah */
+		SZ_READ_VINT(sz);
+
+		switch (t) {
+		case kEmptyStream:
+		case kEmptyFile:
+		case kAnti: /* AntiFile, OMFG */
+					/* We don't care about these bits */
+		case kCTime:
+		case kATime:
+		case kMTime:
+			/* We don't care of these guys, but we still have to parse them, gah */
+			if (sz > 0) {
+				SZ_SKIP_BYTES(sz);
+			}
+			break;
+		case kName:
+			/* The most useful part in this whole bloody format */
+			b = *p; /* External flag */
+			SZ_SKIP_BYTES(1);
+
+			if (b) {
+				/* TODO: for the god sake, do something about external
+				 * filenames...
+				 */
+				guint64 tmp;
+
+				SZ_READ_VINT(tmp);
+			}
+			else {
+				for (i = 0; i < nfiles; i++) {
+					/* Zero terminated wchar_t: happy converting... */
+					/* First, find terminator */
+					const guchar *fend = NULL, *tp = p;
+					GString *res;
+
+					while (tp < end - 1) {
+						if (*tp == 0 && *(tp + 1) == 0) {
+							fend = tp;
+							break;
+						}
+
+						tp += 2;
+					}
+
+					if (fend == NULL || fend - p == 0) {
+						/* Crap instead of fname */
+						msg_debug_archive("bad 7zip name; %s", G_STRLOC);
+						goto end;
+					}
+
+					res = rspamd_7zip_ucs2_to_utf8(task, p, fend);
+
+					if (res != NULL) {
+						fentry = g_malloc0(sizeof(*fentry));
+						fentry->fname = res;
+						g_ptr_array_add(arch->files, fentry);
+						msg_debug_archive("7zip: found file %v", res);
+					}
+					else {
+						msg_debug_archive("bad 7zip name; %s", G_STRLOC);
+					}
+					/* Skip zero terminating character */
+					p = fend + 2;
+				}
+			}
+			break;
+		case kDummy:
+		case kWinAttributes:
+			if (sz > 0) {
+				SZ_SKIP_BYTES(sz);
+			}
+			break;
+		default:
+			p = NULL;
+			msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+			goto end;
+			break;
+		}
+	}
+
+end:
+	return p;
+}
+
+static const guchar *
+rspamd_7zip_read_next_section(struct rspamd_task *task,
+							  const guchar *p, const guchar *end,
+							  struct rspamd_archive *arch)
+{
+	guchar t = *p;
+
+	SZ_SKIP_BYTES(1);
+
+	msg_debug_archive("7zip: read section %xc", t);
+
+	switch (t) {
+	case kHeader:
+		/* We just skip byte and go further */
+		break;
+	case kEncodedHeader:
+		/*
+		 * In fact, headers are just packed, but we assume it as
+		 * encrypted to distinguish from the normal archives
+		 */
+		msg_debug_archive("7zip: encoded header, needs to be uncompressed");
+		arch->flags |= RSPAMD_ARCHIVE_CANNOT_READ;
+		p = NULL; /* Cannot get anything useful */
+		break;
+	case kArchiveProperties:
+		p = rspamd_7zip_read_archive_props(task, p, end, arch);
+		break;
+	case kMainStreamsInfo:
+		p = rspamd_7zip_read_main_streams_info(task, p, end, arch);
+		break;
+	case kAdditionalStreamsInfo:
+		p = rspamd_7zip_read_main_streams_info(task, p, end, arch);
+		break;
+	case kFilesInfo:
+		p = rspamd_7zip_read_files_info(task, p, end, arch);
+		break;
+	case kEnd:
+		p = NULL;
+		msg_debug_archive("7zip: read final section");
+		break;
+	default:
+		p = NULL;
+		msg_debug_archive("bad 7zip type: %xc; %s", t, G_STRLOC);
+		break;
+	}
+
+	return p;
+}
+
+static void
+rspamd_archive_process_7zip(struct rspamd_task *task,
+							struct rspamd_mime_part *part)
+{
+	struct rspamd_archive *arch;
+	const guchar *start, *p, *end;
+	const guchar sz_magic[] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C};
+	guint64 section_offset = 0, section_length = 0;
+
+	start = part->parsed_data.begin;
+	p = start;
+	end = p + part->parsed_data.len;
+
+	if (end - p <= sizeof(guint64) + sizeof(guint32) ||
+		memcmp(p, sz_magic, sizeof(sz_magic)) != 0) {
+		msg_debug_archive("7z archive is invalid (no 7z magic)");
+
+		return;
+	}
+
+	arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+	arch->files = g_ptr_array_new();
+	arch->type = RSPAMD_ARCHIVE_7ZIP;
+	rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+								  arch);
+
+	/* Magic (6 bytes) + version (2 bytes) + crc32 (4 bytes) */
+	p += sizeof(guint64) + sizeof(guint32);
+
+	SZ_READ_UINT64(section_offset);
+	SZ_READ_UINT64(section_length);
+
+	if (end - p > sizeof(guint32)) {
+		p += sizeof(guint32);
+	}
+	else {
+		msg_debug_archive("7z archive is invalid (truncated crc)");
+
+		return;
+	}
+
+	if (end - p > section_offset) {
+		p += section_offset;
+	}
+	else {
+		msg_debug_archive("7z archive is invalid (incorrect section offset)");
+
+		return;
+	}
+
+	while ((p = rspamd_7zip_read_next_section(task, p, end, arch)) != NULL)
+		;
+
+	part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+	part->specific.arch = arch;
+	if (part->cd != NULL) {
+		arch->archive_name = &part->cd->filename;
+	}
+	arch->size = part->parsed_data.len;
+}
+
+static void
+rspamd_archive_process_gzip(struct rspamd_task *task,
+							struct rspamd_mime_part *part)
+{
+	struct rspamd_archive *arch;
+	const guchar *start, *p, *end;
+	const guchar gz_magic[] = {0x1F, 0x8B};
+	guchar flags;
+
+	start = part->parsed_data.begin;
+	p = start;
+	end = p + part->parsed_data.len;
+
+	if (end - p <= 10 || memcmp(p, gz_magic, sizeof(gz_magic)) != 0) {
+		msg_debug_archive("gzip archive is invalid (no gzip magic)");
+
+		return;
+	}
+
+	arch = rspamd_mempool_alloc0(task->task_pool, sizeof(*arch));
+	arch->files = g_ptr_array_sized_new(1);
+	arch->type = RSPAMD_ARCHIVE_GZIP;
+	if (part->cd) {
+		arch->archive_name = &part->cd->filename;
+	}
+	rspamd_mempool_add_destructor(task->task_pool, rspamd_archive_dtor,
+								  arch);
+
+	flags = p[3];
+
+	if (flags & (1u << 5)) {
+		arch->flags |= RSPAMD_ARCHIVE_ENCRYPTED;
+	}
+
+	if (flags & (1u << 3)) {
+		/* We have file name presented in archive, try to use it */
+		if (flags & (1u << 1)) {
+			/* Multipart */
+			p += 12;
+		}
+		else {
+			p += 10;
+		}
+
+		if (flags & (1u << 2)) {
+			/* Optional section */
+			guint16 optlen = 0;
+
+			RAR_READ_UINT16(optlen);
+
+			if (end <= p + optlen) {
+				msg_debug_archive("gzip archive is invalid, bad extra length: %d",
+								  (int) optlen);
+
+				return;
+			}
+
+			p += optlen;
+		}
+
+		/* Read file name */
+		const guchar *fname_start = p;
+
+		while (p < end) {
+			if (*p == '\0') {
+				if (p > fname_start) {
+					struct rspamd_archive_file *f;
+
+					f = g_malloc0(sizeof(*f));
+
+					rspamd_archive_file_try_utf(task, arch, f,
+												fname_start, p - fname_start);
+
+					if (f->fname) {
+						g_ptr_array_add(arch->files, f);
+
+						if (f->flags & RSPAMD_ARCHIVE_FILE_OBFUSCATED) {
+							arch->flags |= RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES;
+						}
+					}
+					else {
+						/* Invalid filename, skip */
+						g_free(f);
+					}
+
+					goto set;
+				}
+			}
+
+			p++;
+		}
+
+		/* Wrong filename, not zero terminated */
+		msg_debug_archive("gzip archive is invalid, bad filename at pos %d",
+						  (int) (p - start));
+
+		return;
+	}
+
+	/* Fallback, we need to extract file name from archive name if possible */
+	if (part->cd && part->cd->filename.len > 0) {
+		const gchar *dot_pos, *slash_pos;
+
+		dot_pos = rspamd_memrchr(part->cd->filename.begin, '.',
+								 part->cd->filename.len);
+
+		if (dot_pos) {
+			struct rspamd_archive_file *f;
+
+			slash_pos = rspamd_memrchr(part->cd->filename.begin, '/',
+									   part->cd->filename.len);
+
+			if (slash_pos && slash_pos < dot_pos) {
+				f = g_malloc0(sizeof(*f));
+				f->fname = g_string_sized_new(dot_pos - slash_pos);
+				g_string_append_len(f->fname, slash_pos + 1,
+									dot_pos - slash_pos - 1);
+
+				msg_debug_archive("fallback to gzip filename based on cd: %v",
+								  f->fname);
+
+				g_ptr_array_add(arch->files, f);
+
+				goto set;
+			}
+			else {
+				const gchar *fname_start = part->cd->filename.begin;
+
+				f = g_malloc0(sizeof(*f));
+
+				if (memchr(fname_start, '.', part->cd->filename.len) != dot_pos) {
+					/* Double dots, something like foo.exe.gz */
+					f->fname = g_string_sized_new(dot_pos - fname_start);
+					g_string_append_len(f->fname, fname_start,
+										dot_pos - fname_start);
+				}
+				else {
+					/* Single dot, something like foo.gzz */
+					f->fname = g_string_sized_new(part->cd->filename.len);
+					g_string_append_len(f->fname, fname_start,
+										part->cd->filename.len);
+				}
+
+				msg_debug_archive("fallback to gzip filename based on cd: %v",
+								  f->fname);
+
+				g_ptr_array_add(arch->files, f);
+
+				goto set;
+			}
+		}
+	}
+
+	return;
+
+set:
+	/* Set archive data */
+	part->part_type = RSPAMD_MIME_PART_ARCHIVE;
+	part->specific.arch = arch;
+	arch->size = part->parsed_data.len;
+}
+
+static gboolean
+rspamd_archive_cheat_detect(struct rspamd_mime_part *part, const gchar *str,
+							const guchar *magic_start, gsize magic_len)
+{
+	struct rspamd_content_type *ct;
+	const gchar *p;
+	rspamd_ftok_t srch, *fname;
+
+	ct = part->ct;
+	RSPAMD_FTOK_ASSIGN(&srch, "application");
+
+	if (ct && ct->type.len && ct->subtype.len > 0 && rspamd_ftok_cmp(&ct->type, &srch) == 0) {
+		if (rspamd_substring_search_caseless(ct->subtype.begin, ct->subtype.len,
+											 str, strlen(str)) != -1) {
+			/* We still need to check magic, see #1848 */
+			if (magic_start != NULL) {
+				if (part->parsed_data.len > magic_len &&
+					memcmp(part->parsed_data.begin,
+						   magic_start, magic_len) == 0) {
+					return TRUE;
+				}
+				/* No magic, refuse this type of archive */
+				return FALSE;
+			}
+			else {
+				return TRUE;
+			}
+		}
+	}
+
+	if (part->cd) {
+		fname = &part->cd->filename;
+
+		if (fname && fname->len > strlen(str)) {
+			p = fname->begin + fname->len - strlen(str);
+
+			if (rspamd_lc_cmp(p, str, strlen(str)) == 0) {
+				if (*(p - 1) == '.') {
+					if (magic_start != NULL) {
+						if (part->parsed_data.len > magic_len &&
+							memcmp(part->parsed_data.begin,
+								   magic_start, magic_len) == 0) {
+							return TRUE;
+						}
+						/* No magic, refuse this type of archive */
+						return FALSE;
+					}
+
+					return TRUE;
+				}
+			}
+		}
+
+		if (magic_start != NULL) {
+			if (part->parsed_data.len > magic_len &&
+				memcmp(part->parsed_data.begin, magic_start, magic_len) == 0) {
+				return TRUE;
+			}
+		}
+	}
+	else {
+		if (magic_start != NULL) {
+			if (part->parsed_data.len > magic_len &&
+				memcmp(part->parsed_data.begin, magic_start, magic_len) == 0) {
+				return TRUE;
+			}
+		}
+	}
+
+	return FALSE;
+}
+
+void rspamd_archives_process(struct rspamd_task *task)
+{
+	guint i;
+	struct rspamd_mime_part *part;
+	const guchar rar_magic[] = {0x52, 0x61, 0x72, 0x21, 0x1A, 0x07};
+	const guchar zip_magic[] = {0x50, 0x4b, 0x03, 0x04};
+	const guchar sz_magic[] = {'7', 'z', 0xBC, 0xAF, 0x27, 0x1C};
+	const guchar gz_magic[] = {0x1F, 0x8B, 0x08};
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+	{
+		if (part->part_type == RSPAMD_MIME_PART_UNDEFINED) {
+			if (part->parsed_data.len > 0) {
+				if (rspamd_archive_cheat_detect(part, "zip",
+												zip_magic, sizeof(zip_magic))) {
+					rspamd_archive_process_zip(task, part);
+				}
+				else if (rspamd_archive_cheat_detect(part, "rar",
+													 rar_magic, sizeof(rar_magic))) {
+					rspamd_archive_process_rar(task, part);
+				}
+				else if (rspamd_archive_cheat_detect(part, "7z",
+													 sz_magic, sizeof(sz_magic))) {
+					rspamd_archive_process_7zip(task, part);
+				}
+				else if (rspamd_archive_cheat_detect(part, "gz",
+													 gz_magic, sizeof(gz_magic))) {
+					rspamd_archive_process_gzip(task, part);
+				}
+
+				if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT) &&
+					part->part_type == RSPAMD_MIME_PART_ARCHIVE &&
+					part->specific.arch) {
+					struct rspamd_archive *arch = part->specific.arch;
+
+					msg_info_task("found %s archive with incorrect content-type: %T/%T",
+								  rspamd_archive_type_str(arch->type),
+								  &part->ct->type, &part->ct->subtype);
+
+					if (!(part->ct->flags & RSPAMD_CONTENT_TYPE_MISSING)) {
+						part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+					}
+				}
+			}
+		}
+	}
+}
+
+
+const gchar *
+rspamd_archive_type_str(enum rspamd_archive_type type)
+{
+	const gchar *ret = "unknown";
+
+	switch (type) {
+	case RSPAMD_ARCHIVE_ZIP:
+		ret = "zip";
+		break;
+	case RSPAMD_ARCHIVE_RAR:
+		ret = "rar";
+		break;
+	case RSPAMD_ARCHIVE_7ZIP:
+		ret = "7z";
+		break;
+	case RSPAMD_ARCHIVE_GZIP:
+		ret = "gz";
+		break;
+	}
+
+	return ret;
+}
diff --git a/src/libmime/archives.h b/src/libmime/archives.h
new file mode 100644
index 0000000..56beb62
--- /dev/null
+++ b/src/libmime/archives.h
@@ -0,0 +1,72 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_ARCHIVES_H_
+#define SRC_LIBMIME_ARCHIVES_H_
+
+#include "config.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_archive_type {
+	RSPAMD_ARCHIVE_ZIP,
+	RSPAMD_ARCHIVE_RAR,
+	RSPAMD_ARCHIVE_7ZIP,
+	RSPAMD_ARCHIVE_GZIP,
+};
+
+enum rspamd_archive_flags {
+	RSPAMD_ARCHIVE_ENCRYPTED = (1u << 0u),
+	RSPAMD_ARCHIVE_CANNOT_READ = (1u << 1u),
+	RSPAMD_ARCHIVE_HAS_OBFUSCATED_FILES = (1u << 2u),
+};
+
+enum rspamd_archive_file_flags {
+	RSPAMD_ARCHIVE_FILE_ENCRYPTED = (1u << 0u),
+	RSPAMD_ARCHIVE_FILE_OBFUSCATED = (1u << 1u),
+};
+
+struct rspamd_archive_file {
+	GString *fname;
+	gsize compressed_size;
+	gsize uncompressed_size;
+	enum rspamd_archive_file_flags flags;
+};
+
+struct rspamd_archive {
+	enum rspamd_archive_type type;
+	const rspamd_ftok_t *archive_name;
+	gsize size;
+	enum rspamd_archive_flags flags;
+	GPtrArray *files; /* Array of struct rspamd_archive_file */
+};
+
+/**
+ * Process archives from a worker task
+ */
+void rspamd_archives_process(struct rspamd_task *task);
+
+/**
+ * Get textual representation of an archive's type
+ */
+const gchar *rspamd_archive_type_str(enum rspamd_archive_type type);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_ARCHIVES_H_ */
diff --git a/src/libmime/content_type.c b/src/libmime/content_type.c
new file mode 100644
index 0000000..765cb87
--- /dev/null
+++ b/src/libmime/content_type.c
@@ -0,0 +1,884 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "libmime/content_type.h"
+#include "smtp_parsers.h"
+#include "utlist.h"
+#include "libserver/url.h"
+#include "libmime/mime_encoding.h"
+
+static gboolean
+rspamd_rfc2231_decode(rspamd_mempool_t *pool,
+					  struct rspamd_content_type_param *param,
+					  gchar *value_start, gchar *value_end)
+{
+	gchar *quote_pos;
+
+	quote_pos = memchr(value_start, '\'', value_end - value_start);
+
+	if (quote_pos == NULL) {
+		/* Plain percent encoding */
+		gsize r = rspamd_url_decode(value_start, value_start,
+									value_end - value_start);
+		param->value.begin = value_start;
+		param->value.len = r;
+	}
+	else {
+		/*
+		 * We can have encoding'language'data, or
+		 * encoding'data (in theory).
+		 * Try to handle both...
+		 */
+		const gchar *charset = NULL;
+		rspamd_ftok_t ctok;
+
+		ctok.begin = value_start;
+		ctok.len = quote_pos - value_start;
+
+		if (ctok.len > 0) {
+			charset = rspamd_mime_detect_charset(&ctok, pool);
+		}
+
+		/* Now, we can check for either next quote sign or, eh, ignore that */
+		value_start = quote_pos + 1;
+
+		quote_pos = memchr(value_start, '\'', value_end - value_start);
+
+		if (quote_pos) {
+			/* Ignore language */
+			value_start = quote_pos + 1;
+		}
+
+		/* Perform percent decoding */
+		gsize r = rspamd_url_decode(value_start, value_start,
+									value_end - value_start);
+		GError *err = NULL;
+
+		if (charset == NULL) {
+			/* Try heuristic */
+			charset = rspamd_mime_charset_find_by_content(value_start, r, TRUE);
+		}
+
+		if (charset == NULL) {
+			msg_warn_pool("cannot convert parameter from charset %T", &ctok);
+
+			return FALSE;
+		}
+
+		param->value.begin = rspamd_mime_text_to_utf8(pool,
+													  value_start, r,
+													  charset, &param->value.len, &err);
+
+		if (param->value.begin == NULL) {
+			msg_warn_pool("cannot convert parameter from charset %s: %e",
+						  charset, err);
+
+			if (err) {
+				g_error_free(err);
+			}
+
+			return FALSE;
+		}
+	}
+
+	param->flags |= RSPAMD_CONTENT_PARAM_RFC2231;
+
+	return TRUE;
+}
+
+static gboolean
+rspamd_param_maybe_rfc2231_process(rspamd_mempool_t *pool,
+								   struct rspamd_content_type_param *param,
+								   gchar *name_start, gchar *name_end,
+								   gchar *value_start, gchar *value_end)
+{
+	const gchar *star_pos;
+
+	star_pos = memchr(name_start, '*', name_end - name_start);
+
+	if (star_pos == NULL) {
+		return FALSE;
+	}
+
+	/* We have three possibilities here:
+	 * 1. name* (just name + 2231 encoding)
+	 * 2. name*(\d+) (piecewise stuff but no rfc2231 encoding)
+	 * 3. name*(\d+)* (piecewise stuff and rfc2231 encoding)
+	 */
+
+	if (star_pos == name_end - 1) {
+		/* First */
+		if (rspamd_rfc2231_decode(pool, param, value_start, value_end)) {
+			param->name.begin = name_start;
+			param->name.len = name_end - name_start - 1;
+		}
+	}
+	else if (*(name_end - 1) == '*') {
+		/* Third */
+		/* Check number */
+		gulong tmp;
+
+		if (!rspamd_strtoul(star_pos + 1, name_end - star_pos - 2, &tmp)) {
+			return FALSE;
+		}
+
+		param->flags |= RSPAMD_CONTENT_PARAM_PIECEWISE | RSPAMD_CONTENT_PARAM_RFC2231;
+		param->rfc2231_id = tmp;
+		param->name.begin = name_start;
+		param->name.len = star_pos - name_start;
+		param->value.begin = value_start;
+		param->value.len = value_end - value_start;
+
+		/* Deal with that later... */
+	}
+	else {
+		/* Second case */
+		gulong tmp;
+
+		if (!rspamd_strtoul(star_pos + 1, name_end - star_pos - 1, &tmp)) {
+			return FALSE;
+		}
+
+		param->flags |= RSPAMD_CONTENT_PARAM_PIECEWISE;
+		param->rfc2231_id = tmp;
+		param->name.begin = name_start;
+		param->name.len = star_pos - name_start;
+		param->value.begin = value_start;
+		param->value.len = value_end - value_start;
+	}
+
+	return TRUE;
+}
+
+static gint32
+rspamd_cmp_pieces(struct rspamd_content_type_param *p1, struct rspamd_content_type_param *p2)
+{
+	return p1->rfc2231_id - p2->rfc2231_id;
+}
+
+static void
+rspamd_postprocess_ct_attributes(rspamd_mempool_t *pool,
+								 GHashTable *htb,
+								 void (*proc)(rspamd_mempool_t *, struct rspamd_content_type_param *, gpointer ud),
+								 gpointer procd)
+{
+	GHashTableIter it;
+	gpointer k, v;
+	struct rspamd_content_type_param *param, *sorted, *cur;
+
+	if (htb == NULL) {
+		return;
+	}
+
+	g_hash_table_iter_init(&it, htb);
+
+	while (g_hash_table_iter_next(&it, &k, &v)) {
+		param = (struct rspamd_content_type_param *) v;
+
+		if (param->flags & RSPAMD_CONTENT_PARAM_PIECEWISE) {
+			/* Reconstruct param */
+			gsize tlen = 0;
+			gchar *ndata, *pos;
+
+			sorted = param;
+			DL_SORT(sorted, rspamd_cmp_pieces);
+
+			DL_FOREACH(sorted, cur)
+			{
+				tlen += cur->value.len;
+			}
+
+			ndata = rspamd_mempool_alloc(pool, tlen);
+			pos = ndata;
+
+			DL_FOREACH(sorted, cur)
+			{
+				memcpy(pos, cur->value.begin, cur->value.len);
+				pos += cur->value.len;
+			}
+
+			if (param->flags & RSPAMD_CONTENT_PARAM_RFC2231) {
+				if (!rspamd_rfc2231_decode(pool, param,
+										   ndata, pos)) {
+					param->flags |= RSPAMD_CONTENT_PARAM_BROKEN;
+					param->value.begin = ndata;
+					param->value.len = tlen;
+				}
+			}
+			else {
+				param->value.begin = ndata;
+				param->value.len = tlen;
+			}
+
+			/* Detach from list */
+			param->next = NULL;
+			param->prev = param;
+		}
+
+		gboolean invalid_utf = FALSE;
+
+		if (param->value.begin != NULL && param->value.len > 0) {
+			param->value.begin = rspamd_mime_header_decode(pool, param->value.begin,
+														   param->value.len, &invalid_utf);
+			param->value.len = strlen(param->value.begin);
+		}
+
+		if (invalid_utf) {
+			param->flags |= RSPAMD_CONTENT_PARAM_BROKEN;
+		}
+
+		proc(pool, param, procd);
+	}
+}
+
+static void
+rspamd_content_type_postprocess(rspamd_mempool_t *pool,
+								struct rspamd_content_type_param *param,
+								gpointer ud)
+{
+	rspamd_ftok_t srch;
+	struct rspamd_content_type_param *found = NULL;
+
+	struct rspamd_content_type *ct = (struct rspamd_content_type *) ud;
+
+	RSPAMD_FTOK_ASSIGN(&srch, "charset");
+
+	if (rspamd_ftok_icase_equal(&param->name, &srch)) {
+		/* Adjust charset */
+		found = param;
+		ct->charset.begin = param->value.begin;
+		ct->charset.len = param->value.len;
+	}
+
+	RSPAMD_FTOK_ASSIGN(&srch, "boundary");
+
+	if (rspamd_ftok_icase_equal(&param->name, &srch)) {
+		found = param;
+		gchar *lc_boundary;
+		/* Adjust boundary */
+		lc_boundary = rspamd_mempool_alloc(pool, param->value.len);
+		memcpy(lc_boundary, param->value.begin, param->value.len);
+		rspamd_str_lc(lc_boundary, param->value.len);
+		ct->boundary.begin = lc_boundary;
+		ct->boundary.len = param->value.len;
+		/* Preserve original (case sensitive) boundary */
+		ct->orig_boundary.begin = param->value.begin;
+		ct->orig_boundary.len = param->value.len;
+	}
+
+	if (!found) {
+		RSPAMD_FTOK_ASSIGN(&srch, "name");
+		if (!rspamd_ftok_icase_equal(&param->name, &srch)) {
+			/* Just lowercase */
+			rspamd_str_lc_utf8((gchar *) param->value.begin, param->value.len);
+		}
+	}
+}
+
+static void
+rspamd_content_disposition_postprocess(rspamd_mempool_t *pool,
+									   struct rspamd_content_type_param *param,
+									   gpointer ud)
+{
+	rspamd_ftok_t srch;
+	struct rspamd_content_disposition *cd = (struct rspamd_content_disposition *) ud;
+
+	srch.begin = "filename";
+	srch.len = 8;
+
+	if (rspamd_ftok_icase_equal(&param->name, &srch)) {
+		/* Adjust filename */
+		cd->filename.begin = param->value.begin;
+		cd->filename.len = param->value.len;
+	}
+}
+
+void rspamd_content_type_add_param(rspamd_mempool_t *pool,
+								   struct rspamd_content_type *ct,
+								   gchar *name_start, gchar *name_end,
+								   gchar *value_start, gchar *value_end)
+{
+	struct rspamd_content_type_param *nparam;
+	rspamd_ftok_t srch;
+	struct rspamd_content_type_param *found = NULL;
+
+	g_assert(ct != NULL);
+
+	nparam = rspamd_mempool_alloc0(pool, sizeof(*nparam));
+	rspamd_str_lc(name_start, name_end - name_start);
+
+	if (!rspamd_param_maybe_rfc2231_process(pool, nparam, name_start,
+											name_end, value_start, value_end)) {
+		nparam->name.begin = name_start;
+		nparam->name.len = name_end - name_start;
+		nparam->value.begin = value_start;
+		nparam->value.len = value_end - value_start;
+	}
+
+	srch.begin = nparam->name.begin;
+	srch.len = nparam->name.len;
+
+	if (ct->attrs) {
+		found = g_hash_table_lookup(ct->attrs, &srch);
+	}
+	else {
+		ct->attrs = g_hash_table_new(rspamd_ftok_icase_hash,
+									 rspamd_ftok_icase_equal);
+	}
+
+	if (!found) {
+		DL_APPEND(found, nparam);
+		g_hash_table_insert(ct->attrs, &nparam->name, nparam);
+	}
+	else {
+		DL_APPEND(found, nparam);
+	}
+}
+
+static struct rspamd_content_type *
+rspamd_content_type_parser(gchar *in, gsize len, rspamd_mempool_t *pool)
+{
+	guint obraces = 0, ebraces = 0, qlen = 0;
+	gchar *p, *c, *end, *pname_start = NULL, *pname_end = NULL;
+	struct rspamd_content_type *res = NULL, val;
+	gboolean eqsign_seen = FALSE;
+	enum {
+		parse_type,
+		parse_subtype,
+		parse_after_subtype,
+		parse_param_name,
+		parse_param_after_name,
+		parse_param_value,
+		parse_param_value_after_quote,
+		parse_space,
+		parse_quoted,
+		parse_comment,
+	} state = parse_space,
+	  next_state = parse_type;
+
+	p = in;
+	c = p;
+	end = p + len;
+	memset(&val, 0, sizeof(val));
+	val.cpy = in;
+
+	while (p < end) {
+		switch (state) {
+		case parse_type:
+			if (g_ascii_isspace(*p) || *p == ';') {
+				/* We have type without subtype */
+				val.type.begin = c;
+				val.type.len = p - c;
+				state = parse_after_subtype;
+			}
+			else if (*p == '/') {
+				val.type.begin = c;
+				val.type.len = p - c;
+				state = parse_space;
+				next_state = parse_subtype;
+				p++;
+			}
+			else {
+				p++;
+			}
+			break;
+		case parse_subtype:
+			if (g_ascii_isspace(*p) || *p == ';') {
+				val.subtype.begin = c;
+				val.subtype.len = p - c;
+				state = parse_after_subtype;
+			}
+			else {
+				p++;
+			}
+			break;
+		case parse_after_subtype:
+			if (*p == ';' || g_ascii_isspace(*p)) {
+				p++;
+			}
+			else if (*p == '(') {
+				c = p;
+				state = parse_comment;
+				next_state = parse_param_name;
+				obraces = 1;
+				ebraces = 0;
+				pname_start = NULL;
+				pname_end = NULL;
+				eqsign_seen = FALSE;
+				p++;
+			}
+			else {
+				c = p;
+				state = parse_param_name;
+				pname_start = NULL;
+				pname_end = NULL;
+				eqsign_seen = FALSE;
+			}
+			break;
+		case parse_param_name:
+			if (*p == '=') {
+				pname_start = c;
+				pname_end = p;
+				state = parse_param_after_name;
+				eqsign_seen = TRUE;
+				p++;
+			}
+			else if (g_ascii_isspace(*p)) {
+				pname_start = c;
+				pname_end = p;
+				state = parse_param_after_name;
+			}
+			else {
+				p++;
+			}
+			break;
+		case parse_param_after_name:
+			if (g_ascii_isspace(*p)) {
+				p++;
+			}
+			else if (*p == '=') {
+				if (eqsign_seen) {
+					/* Treat as value start */
+					c = p;
+					eqsign_seen = FALSE;
+					state = parse_param_value;
+					p++;
+				}
+				else {
+					eqsign_seen = TRUE;
+					p++;
+				}
+			}
+			else {
+				if (eqsign_seen) {
+					state = parse_param_value;
+					c = p;
+				}
+				else {
+					/* Invalid parameter without value */
+					c = p;
+					state = parse_param_name;
+					pname_start = NULL;
+					pname_end = NULL;
+				}
+			}
+			break;
+		case parse_param_value:
+			if (*p == '"') {
+				p++;
+				c = p;
+				state = parse_quoted;
+				next_state = parse_param_value_after_quote;
+			}
+			else if (g_ascii_isspace(*p)) {
+				if (pname_start && pname_end && pname_end > pname_start) {
+					rspamd_content_type_add_param(pool, &val, pname_start,
+												  pname_end, c, p);
+				}
+
+				state = parse_space;
+				next_state = parse_param_name;
+				pname_start = NULL;
+				pname_end = NULL;
+			}
+			else if (*p == '(') {
+				if (pname_start && pname_end && pname_end > pname_start) {
+					rspamd_content_type_add_param(pool, &val, pname_start,
+												  pname_end, c, p);
+				}
+
+				obraces = 1;
+				ebraces = 0;
+				p++;
+				state = parse_comment;
+				next_state = parse_param_name;
+				pname_start = NULL;
+				pname_end = NULL;
+			}
+			else if (*p == ';') {
+				if (pname_start && pname_end && pname_end > pname_start) {
+					rspamd_content_type_add_param(pool, &val, pname_start,
+												  pname_end, c, p);
+				}
+
+				p++;
+				state = parse_space;
+				next_state = parse_param_name;
+				pname_start = NULL;
+				pname_end = NULL;
+			}
+			else {
+				p++;
+			}
+			break;
+		case parse_param_value_after_quote:
+			if (pname_start && pname_end && pname_end > pname_start) {
+				rspamd_content_type_add_param(pool, &val, pname_start,
+											  pname_end, c, c + qlen);
+			}
+
+			if (*p == '"') {
+				p++;
+
+				if (p == end) {
+					/* Last quote: done... */
+					state = parse_space;
+					break;
+				}
+
+				if (*p == ';') {
+					p++;
+					state = parse_space;
+					next_state = parse_param_name;
+					pname_start = NULL;
+					pname_end = NULL;
+					continue;
+				}
+			}
+
+			/* We should not normally be here in fact */
+			if (g_ascii_isspace(*p)) {
+				state = parse_space;
+				next_state = parse_param_name;
+				pname_start = NULL;
+				pname_end = NULL;
+			}
+			else if (*p == '(') {
+				obraces = 1;
+				ebraces = 0;
+				p++;
+				state = parse_comment;
+				next_state = parse_param_name;
+				pname_start = NULL;
+				pname_end = NULL;
+			}
+			else {
+				state = parse_param_name;
+				pname_start = NULL;
+				pname_end = NULL;
+				c = p;
+			}
+			break;
+		case parse_quoted:
+			if (*p == '\\') {
+				/* Quoted pair */
+				if (p + 1 < end) {
+					p += 2;
+				}
+				else {
+					p++;
+				}
+			}
+			else if (*p == '"') {
+				qlen = p - c;
+				state = next_state;
+			}
+			else {
+				p++;
+			}
+			break;
+		case parse_comment:
+			if (*p == '(') {
+				obraces++;
+				p++;
+			}
+			else if (*p == ')') {
+				ebraces++;
+				p++;
+
+				if (ebraces == obraces && p < end) {
+					if (g_ascii_isspace(*p)) {
+						state = parse_space;
+					}
+					else {
+						c = p;
+						state = next_state;
+					}
+				}
+			}
+			else {
+				p++;
+			}
+			break;
+		case parse_space:
+			if (g_ascii_isspace(*p)) {
+				p++;
+			}
+			else if (*p == '(') {
+				obraces = 1;
+				ebraces = 0;
+				p++;
+				state = parse_comment;
+			}
+			else {
+				c = p;
+				state = next_state;
+			}
+			break;
+		}
+	}
+
+	/* Process leftover */
+	switch (state) {
+	case parse_type:
+		val.type.begin = c;
+		val.type.len = p - c;
+		break;
+	case parse_subtype:
+		val.subtype.begin = c;
+		val.subtype.len = p - c;
+		break;
+	case parse_param_value:
+		if (pname_start && pname_end && pname_end > pname_start) {
+			if (p > c && *(p - 1) == ';') {
+				p--;
+			}
+
+			rspamd_content_type_add_param(pool, &val, pname_start,
+										  pname_end, c, p);
+		}
+		break;
+	case parse_param_value_after_quote:
+		if (pname_start && pname_end && pname_end > pname_start) {
+			rspamd_content_type_add_param(pool, &val, pname_start,
+										  pname_end, c, c + qlen);
+		}
+		break;
+	default:
+		break;
+	}
+
+	if (val.type.len > 0) {
+		gchar *tmp;
+
+		res = rspamd_mempool_alloc(pool, sizeof(val));
+		memcpy(res, &val, sizeof(val));
+
+		/*
+		 * Lowercase type and subtype as they are specified as case insensitive
+		 * in rfc2045 section 5.1
+		 */
+		tmp = rspamd_mempool_alloc(pool, val.type.len);
+		memcpy(tmp, val.type.begin, val.type.len);
+		rspamd_str_lc(tmp, val.type.len);
+		res->type.begin = tmp;
+
+		if (val.subtype.len > 0) {
+			tmp = rspamd_mempool_alloc(pool, val.subtype.len);
+			memcpy(tmp, val.subtype.begin, val.subtype.len);
+			rspamd_str_lc(tmp, val.subtype.len);
+			res->subtype.begin = tmp;
+		}
+	}
+
+	return res;
+}
+
+struct rspamd_content_type *
+rspamd_content_type_parse(const gchar *in,
+						  gsize len, rspamd_mempool_t *pool)
+{
+	struct rspamd_content_type *res = NULL;
+	rspamd_ftok_t srch;
+	gchar *cpy;
+
+	cpy = rspamd_mempool_alloc(pool, len + 1);
+	rspamd_strlcpy(cpy, in, len + 1);
+
+	if ((res = rspamd_content_type_parser(cpy, len, pool)) != NULL) {
+		if (res->attrs) {
+			rspamd_mempool_add_destructor(pool,
+										  (rspamd_mempool_destruct_t) g_hash_table_unref, res->attrs);
+
+			rspamd_postprocess_ct_attributes(pool, res->attrs,
+											 rspamd_content_type_postprocess, res);
+		}
+
+		/* Now do some hacks to work with broken content types */
+		if (res->subtype.len == 0) {
+			res->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+			RSPAMD_FTOK_ASSIGN(&srch, "text");
+
+			if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+				/* Workaround for Content-Type: text */
+				/* Assume text/plain */
+				RSPAMD_FTOK_ASSIGN(&srch, "plain");
+			}
+			else {
+				RSPAMD_FTOK_ASSIGN(&srch, "html");
+
+				if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+					/* Workaround for Content-Type: html */
+					RSPAMD_FTOK_ASSIGN(&res->type, "text");
+					RSPAMD_FTOK_ASSIGN(&res->subtype, "html");
+				}
+				else {
+					RSPAMD_FTOK_ASSIGN(&srch, "application");
+
+					if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+						RSPAMD_FTOK_ASSIGN(&res->subtype, "octet-stream");
+					}
+				}
+			}
+		}
+		else {
+			/* Common mistake done by retards */
+			RSPAMD_FTOK_ASSIGN(&srch, "alternate");
+
+			if (rspamd_ftok_casecmp(&res->subtype, &srch) == 0) {
+				res->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+				RSPAMD_FTOK_ASSIGN(&res->subtype, "alternative");
+			}
+
+			/* PKCS7 smime */
+			RSPAMD_FTOK_ASSIGN(&srch, "pkcs7-mime");
+			if (rspamd_substring_search(res->subtype.begin, res->subtype.len,
+										srch.begin, srch.len) != -1) {
+				res->flags |= RSPAMD_CONTENT_TYPE_SMIME;
+			}
+		}
+
+		RSPAMD_FTOK_ASSIGN(&srch, "multipart");
+
+		if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+			res->flags |= RSPAMD_CONTENT_TYPE_MULTIPART;
+
+			RSPAMD_FTOK_ASSIGN(&srch, "encrypted");
+			if (rspamd_ftok_casecmp(&res->subtype, &srch) == 0) {
+				res->flags |= RSPAMD_CONTENT_TYPE_ENCRYPTED;
+			}
+		}
+		else {
+			RSPAMD_FTOK_ASSIGN(&srch, "text");
+
+			if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+				res->flags |= RSPAMD_CONTENT_TYPE_TEXT;
+			}
+			else {
+				RSPAMD_FTOK_ASSIGN(&srch, "message");
+
+				if (rspamd_ftok_casecmp(&res->type, &srch) == 0) {
+					RSPAMD_FTOK_ASSIGN(&srch, "delivery-status");
+
+					if (rspamd_ftok_casecmp(&res->subtype, &srch) == 0) {
+						res->flags |= RSPAMD_CONTENT_TYPE_TEXT | RSPAMD_CONTENT_TYPE_DSN;
+					}
+					else {
+						RSPAMD_FTOK_ASSIGN(&srch, "notification");
+
+						if (rspamd_substring_search_caseless(res->subtype.begin,
+															 res->subtype.len, srch.begin, srch.len) != -1) {
+							res->flags |= RSPAMD_CONTENT_TYPE_TEXT |
+										  RSPAMD_CONTENT_TYPE_DSN;
+						}
+						else {
+							res->flags |= RSPAMD_CONTENT_TYPE_MESSAGE;
+						}
+					}
+				}
+			}
+		}
+	}
+	else {
+		msg_warn_pool("cannot parse content type: %*s", (gint) len, cpy);
+	}
+
+	return res;
+}
+
+void rspamd_content_disposition_add_param(rspamd_mempool_t *pool,
+										  struct rspamd_content_disposition *cd,
+										  const gchar *name_start, const gchar *name_end,
+										  const gchar *value_start, const gchar *value_end)
+{
+	rspamd_ftok_t srch;
+	gchar *name_cpy, *value_cpy, *name_cpy_end, *value_cpy_end;
+	struct rspamd_content_type_param *found = NULL, *nparam;
+
+	g_assert(cd != NULL);
+
+	name_cpy = rspamd_mempool_alloc(pool, name_end - name_start);
+	memcpy(name_cpy, name_start, name_end - name_start);
+	name_cpy_end = name_cpy + (name_end - name_start);
+
+	value_cpy = rspamd_mempool_alloc(pool, value_end - value_start);
+	memcpy(value_cpy, value_start, value_end - value_start);
+	value_cpy_end = value_cpy + (value_end - value_start);
+
+	nparam = rspamd_mempool_alloc0(pool, sizeof(*nparam));
+	rspamd_str_lc(name_cpy, name_cpy_end - name_cpy);
+
+	if (!rspamd_param_maybe_rfc2231_process(pool, nparam, name_cpy,
+											name_cpy_end, value_cpy, value_cpy_end)) {
+		nparam->name.begin = name_cpy;
+		nparam->name.len = name_cpy_end - name_cpy;
+		nparam->value.begin = value_cpy;
+		nparam->value.len = value_cpy_end - value_cpy;
+	}
+
+	srch.begin = nparam->name.begin;
+	srch.len = nparam->name.len;
+
+	if (cd->attrs) {
+		found = g_hash_table_lookup(cd->attrs, &srch);
+	}
+	else {
+		cd->attrs = g_hash_table_new(rspamd_ftok_icase_hash,
+									 rspamd_ftok_icase_equal);
+	}
+
+	if (!found) {
+		DL_APPEND(found, nparam);
+		g_hash_table_insert(cd->attrs, &nparam->name, nparam);
+	}
+	else {
+		DL_APPEND(found, nparam);
+	}
+}
+
+struct rspamd_content_disposition *
+rspamd_content_disposition_parse(const gchar *in,
+								 gsize len, rspamd_mempool_t *pool)
+{
+	struct rspamd_content_disposition *res = NULL, val;
+
+	if (rspamd_content_disposition_parser(in, len, &val, pool)) {
+
+		if (val.type == RSPAMD_CT_UNKNOWN) {
+			/* 'Fix' type to attachment as MUA does */
+			val.type = RSPAMD_CT_ATTACHMENT;
+		}
+
+		res = rspamd_mempool_alloc(pool, sizeof(val));
+		memcpy(res, &val, sizeof(val));
+		res->lc_data = rspamd_mempool_alloc(pool, len + 1);
+		rspamd_strlcpy(res->lc_data, in, len + 1);
+		rspamd_str_lc(res->lc_data, len);
+
+		if (res->attrs) {
+			rspamd_postprocess_ct_attributes(pool, res->attrs,
+											 rspamd_content_disposition_postprocess, res);
+			rspamd_mempool_add_destructor(pool,
+										  (rspamd_mempool_destruct_t) g_hash_table_unref, res->attrs);
+		}
+	}
+	else {
+		msg_warn_pool("cannot parse content disposition: %*s",
+					  (gint) len, in);
+	}
+
+	return res;
+}
diff --git a/src/libmime/content_type.h b/src/libmime/content_type.h
new file mode 100644
index 0000000..ac49bdc
--- /dev/null
+++ b/src/libmime/content_type.h
@@ -0,0 +1,130 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_CONTENT_TYPE_H_
+#define SRC_LIBMIME_CONTENT_TYPE_H_
+
+#include "config.h"
+#include "libutil/fstring.h"
+#include "libutil/mem_pool.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+enum rspamd_content_type_flags {
+	RSPAMD_CONTENT_TYPE_VALID = 0,
+	RSPAMD_CONTENT_TYPE_BROKEN = 1 << 0,
+	RSPAMD_CONTENT_TYPE_MULTIPART = 1 << 1,
+	RSPAMD_CONTENT_TYPE_TEXT = 1 << 2,
+	RSPAMD_CONTENT_TYPE_MESSAGE = 1 << 3,
+	RSPAMD_CONTENT_TYPE_DSN = 1 << 4,
+	RSPAMD_CONTENT_TYPE_MISSING = 1 << 5,
+	RSPAMD_CONTENT_TYPE_ENCRYPTED = 1 << 6,
+	RSPAMD_CONTENT_TYPE_SMIME = 1 << 7,
+};
+
+enum rspamd_content_param_flags {
+	RSPAMD_CONTENT_PARAM_NORMAL = 0,
+	RSPAMD_CONTENT_PARAM_RFC2231 = (1 << 0),
+	RSPAMD_CONTENT_PARAM_PIECEWISE = (1 << 1),
+	RSPAMD_CONTENT_PARAM_BROKEN = (1 << 2),
+};
+
+struct rspamd_content_type_param {
+	rspamd_ftok_t name;
+	rspamd_ftok_t value;
+	guint rfc2231_id;
+	enum rspamd_content_param_flags flags;
+	struct rspamd_content_type_param *prev, *next;
+};
+
+struct rspamd_content_type {
+	gchar *cpy;
+	rspamd_ftok_t type;
+	rspamd_ftok_t subtype;
+	rspamd_ftok_t charset;
+	rspamd_ftok_t boundary;
+	rspamd_ftok_t orig_boundary;
+	enum rspamd_content_type_flags flags;
+	GHashTable *attrs; /* Can be empty */
+};
+
+enum rspamd_content_disposition_type {
+	RSPAMD_CT_UNKNOWN = 0,
+	RSPAMD_CT_INLINE = 1,
+	RSPAMD_CT_ATTACHMENT = 2,
+};
+
+struct rspamd_content_disposition {
+	gchar *lc_data;
+	enum rspamd_content_disposition_type type;
+	rspamd_ftok_t filename;
+	GHashTable *attrs; /* Can be empty */
+};
+
+/**
+ * Adds new parameter to content type structure
+ * @param ct
+ * @param name_start (can be modified)
+ * @param name_end
+ * @param value_start (can be modified)
+ * @param value_end
+ */
+void rspamd_content_type_add_param(rspamd_mempool_t *pool,
+								   struct rspamd_content_type *ct,
+								   gchar *name_start, gchar *name_end,
+								   gchar *value_start, gchar *value_end);
+
+/**
+ * Parse content type from the header (performs copy + lowercase)
+ * @param in
+ * @param len
+ * @param pool
+ * @return
+ */
+struct rspamd_content_type *rspamd_content_type_parse(const gchar *in,
+													  gsize len, rspamd_mempool_t *pool);
+
+/**
+ * Adds new param for content disposition header
+ * @param pool
+ * @param cd
+ * @param name_start
+ * @param name_end
+ * @param value_start
+ * @param value_end
+ */
+void rspamd_content_disposition_add_param(rspamd_mempool_t *pool,
+										  struct rspamd_content_disposition *cd,
+										  const gchar *name_start, const gchar *name_end,
+										  const gchar *value_start, const gchar *value_end);
+
+/**
+ * Parse content-disposition header
+ * @param in
+ * @param len
+ * @param pool
+ * @return
+ */
+struct rspamd_content_disposition *rspamd_content_disposition_parse(const gchar *in,
+																	gsize len,
+																	rspamd_mempool_t *pool);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_CONTENT_TYPE_H_ */
diff --git a/src/libmime/email_addr.c b/src/libmime/email_addr.c
new file mode 100644
index 0000000..0af7388
--- /dev/null
+++ b/src/libmime/email_addr.c
@@ -0,0 +1,563 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "email_addr.h"
+#include "message.h"
+#include "printf.h"
+#include "smtp_parsers.h"
+
+static void
+rspamd_email_address_unescape(struct rspamd_email_address *addr)
+{
+	const char *h, *end;
+	char *t, *d;
+
+	if (addr->user_len == 0) {
+		return;
+	}
+
+	d = g_malloc(addr->user_len);
+	t = d;
+	h = addr->user;
+	end = h + addr->user_len;
+
+	while (h < end) {
+		if (*h != '\\') {
+			*t++ = *h;
+		}
+		h++;
+	}
+
+	addr->user = d;
+	addr->user_len = t - d;
+	addr->flags |= RSPAMD_EMAIL_ADDR_USER_ALLOCATED;
+}
+
+struct rspamd_email_address *
+rspamd_email_address_from_smtp(const gchar *str, guint len)
+{
+	struct rspamd_email_address addr, *ret;
+	gsize nlen;
+
+	if (str == NULL || len == 0) {
+		return NULL;
+	}
+
+	rspamd_smtp_addr_parse(str, len, &addr);
+
+	if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
+		ret = g_malloc(sizeof(*ret));
+		memcpy(ret, &addr, sizeof(addr));
+
+		if ((ret->flags & RSPAMD_EMAIL_ADDR_QUOTED) && ret->addr[0] == '"') {
+			if (ret->flags & RSPAMD_EMAIL_ADDR_HAS_BACKSLASH) {
+				/* We also need to unquote user */
+				rspamd_email_address_unescape(ret);
+			}
+
+			/* We need to unquote addr */
+			nlen = ret->domain_len + ret->user_len + 2;
+			ret->addr = g_malloc(nlen + 1);
+			ret->addr_len = rspamd_snprintf((char *) ret->addr, nlen, "%*s@%*s",
+											(gint) ret->user_len, ret->user,
+											(gint) ret->domain_len, ret->domain);
+			ret->flags |= RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED;
+		}
+
+		return ret;
+	}
+
+	return NULL;
+}
+
+void rspamd_email_address_free(struct rspamd_email_address *addr)
+{
+	if (addr) {
+		if (addr->flags & RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED) {
+			g_free((void *) addr->addr);
+		}
+
+		if (addr->flags & RSPAMD_EMAIL_ADDR_USER_ALLOCATED) {
+			g_free((void *) addr->user);
+		}
+
+		g_free(addr);
+	}
+}
+
+static inline void
+rspamd_email_address_add(rspamd_mempool_t *pool,
+						 GPtrArray *ar,
+						 struct rspamd_email_address *addr,
+						 GString *name)
+{
+	struct rspamd_email_address *elt;
+	guint nlen;
+
+	elt = g_malloc0(sizeof(*elt));
+	rspamd_mempool_notify_alloc(pool, sizeof(*elt));
+
+	if (addr != NULL) {
+		memcpy(elt, addr, sizeof(*addr));
+	}
+	else {
+		elt->addr = "";
+		elt->domain = "";
+		elt->raw = "<>";
+		elt->raw_len = 2;
+		elt->user = "";
+		elt->flags |= RSPAMD_EMAIL_ADDR_EMPTY;
+	}
+
+	if ((elt->flags & RSPAMD_EMAIL_ADDR_QUOTED) && elt->addr[0] == '"') {
+		if (elt->flags & RSPAMD_EMAIL_ADDR_HAS_BACKSLASH) {
+			/* We also need to unquote user */
+			rspamd_email_address_unescape(elt);
+		}
+
+		/* We need to unquote addr */
+		nlen = elt->domain_len + elt->user_len + 2;
+		elt->addr = g_malloc(nlen + 1);
+		rspamd_mempool_notify_alloc(pool, nlen + 1);
+		elt->addr_len = rspamd_snprintf((char *) elt->addr, nlen, "%*s@%*s",
+										(gint) elt->user_len, elt->user,
+										(gint) elt->domain_len, elt->domain);
+		elt->flags |= RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED;
+	}
+
+	if (name->len > 0) {
+		rspamd_gstring_strip(name, " \t\v");
+		elt->name = rspamd_mime_header_decode(pool, name->str, name->len, NULL);
+	}
+
+	rspamd_mempool_notify_alloc(pool, name->len);
+	g_ptr_array_add(ar, elt);
+}
+
+/*
+ * Tries to parse an email address that doesn't conform RFC
+ */
+static gboolean
+rspamd_email_address_parse_heuristic(const char *data, size_t len,
+									 struct rspamd_email_address *addr)
+{
+	const gchar *p = data, *at = NULL, *end = data + len;
+	gboolean ret = FALSE;
+
+	memset(addr, 0, sizeof(*addr));
+
+	if (*p == '<' && len > 1) {
+		/* Angled address */
+		addr->addr_len = rspamd_memcspn(p + 1, ">", len - 1);
+		addr->addr = p + 1;
+		addr->raw = p;
+		addr->raw_len = len;
+		ret = TRUE;
+
+		p = p + 1;
+		len = addr->addr_len;
+		end = p + len;
+	}
+	else if (len > 0) {
+		addr->addr = p;
+		addr->addr_len = len;
+		addr->raw = p;
+		addr->raw_len = len;
+		ret = TRUE;
+	}
+
+	if (ret) {
+		at = rspamd_memrchr(p, '@', len);
+
+		if (at != NULL && at + 1 < end) {
+			addr->domain = at + 1;
+			addr->domain_len = end - (at + 1);
+			addr->user = p;
+			addr->user_len = at - p;
+		}
+
+		if (rspamd_str_has_8bit(p, len)) {
+			addr->flags |= RSPAMD_EMAIL_ADDR_HAS_8BIT;
+		}
+	}
+
+	return ret;
+}
+
+static inline int
+rspamd_email_address_check_and_add(const gchar *start, gsize len,
+								   GPtrArray *res,
+								   rspamd_mempool_t *pool,
+								   GString *ns,
+								   gint max_elements)
+{
+	struct rspamd_email_address addr;
+
+	g_assert(res != NULL);
+
+	if (max_elements > 0 && res->len >= max_elements) {
+		msg_info_pool_check("reached maximum number of elements %d when adding %v",
+							max_elements,
+							ns);
+
+		return -1;
+	}
+
+	/* The whole email is likely address */
+	memset(&addr, 0, sizeof(addr));
+	rspamd_smtp_addr_parse(start, len, &addr);
+
+	if (addr.flags & RSPAMD_EMAIL_ADDR_VALID) {
+		rspamd_email_address_add(pool, res, &addr, ns);
+	}
+	else {
+		/* Try heuristic */
+		if (rspamd_email_address_parse_heuristic(start,
+												 len, &addr)) {
+			rspamd_email_address_add(pool, res, &addr, ns);
+
+			return 1;
+		}
+		else {
+			return 0;
+		}
+	}
+
+	return 1;
+}
+
+GPtrArray *
+rspamd_email_address_from_mime(rspamd_mempool_t *pool, const gchar *hdr,
+							   guint len,
+							   GPtrArray *src,
+							   gint max_elements)
+{
+	GPtrArray *res = src;
+	gboolean seen_at = FALSE, seen_obrace = FALSE;
+
+	const gchar *p = hdr, *end = hdr + len, *c = hdr, *t;
+	GString *ns, *cpy;
+	gint obraces, ebraces;
+	enum {
+		parse_name = 0,
+		parse_quoted,
+		parse_addr,
+		skip_spaces
+	} state = parse_name,
+	  next_state = parse_name;
+
+	if (res == NULL) {
+		res = g_ptr_array_sized_new(2);
+		rspamd_mempool_add_destructor(pool, rspamd_email_address_list_destroy,
+									  res);
+	}
+	else if (max_elements > 0 && res->len >= max_elements) {
+		msg_info_pool_check("reached maximum number of elements %d", max_elements);
+
+		return res;
+	}
+
+	ns = g_string_sized_new(len);
+	cpy = g_string_sized_new(len);
+
+	rspamd_mempool_add_destructor(pool, rspamd_gstring_free_hard, cpy);
+
+	/* First, we need to remove all comments as they are terrible */
+	obraces = 0;
+	ebraces = 0;
+
+	while (p < end) {
+		if (state == parse_name) {
+			if (*p == '\\') {
+				if (obraces == 0) {
+					g_string_append_c(cpy, *p);
+				}
+
+				p++;
+			}
+			else {
+				if (*p == '"') {
+					state = parse_quoted;
+				}
+				else if (*p == '(') {
+					obraces++; /* To avoid ) itself being copied */
+				}
+				else if (*p == ')') {
+					ebraces++;
+					p++;
+				}
+
+				if (obraces == ebraces) {
+					obraces = 0;
+					ebraces = 0;
+				}
+			}
+
+			if (p < end && obraces == 0) {
+				g_string_append_c(cpy, *p);
+			}
+		}
+		else {
+			/* Quoted elt */
+			if (*p == '\\') {
+				g_string_append_c(cpy, *p);
+				p++;
+			}
+			else {
+				if (*p == '"') {
+					state = parse_name;
+				}
+			}
+
+			if (p < end) {
+				g_string_append_c(cpy, *p);
+			}
+		}
+
+		p++;
+	}
+
+	state = parse_name;
+
+	p = cpy->str;
+	c = p;
+	end = p + cpy->len;
+
+	while (p < end) {
+		switch (state) {
+		case parse_name:
+			if (*p == '"') {
+				/* We need to strip last spaces and update `ns` */
+				if (p > c) {
+					guint nspaces = 0;
+
+					t = p - 1;
+
+					while (t > c && g_ascii_isspace(*t)) {
+						t--;
+						nspaces++;
+					}
+
+					g_string_append_len(ns, c, t - c + 1);
+
+					if (nspaces > 0) {
+						g_string_append_c(ns, ' ');
+					}
+				}
+
+				state = parse_quoted;
+				c = p + 1;
+			}
+			else if (*p == '<') {
+				if (p > c) {
+					t = p - 1;
+
+					while (t > c && g_ascii_isspace(*t)) {
+						t--;
+					}
+
+					g_string_append_len(ns, c, t - c + 1);
+				}
+
+				c = p;
+				state = parse_addr;
+			}
+			else if (*p == ',') {
+				if (p > c && seen_at) {
+					/*
+					 * Last token must be the address:
+					 * e.g. Some name name@domain.com
+					 */
+					t = p - 1;
+
+					while (t > c && g_ascii_isspace(*t)) {
+						t--;
+					}
+
+					int check = rspamd_email_address_check_and_add(c, t - c + 1,
+																   res, pool, ns, max_elements);
+
+					if (check == 0 && res->len == 0) {
+						/* Insert fake address */
+						rspamd_email_address_add(pool, res, NULL, ns);
+					}
+					else if (check != 1) {
+						goto end;
+					}
+
+					/* Cleanup for the next use */
+					g_string_set_size(ns, 0);
+					seen_at = FALSE;
+				}
+
+				state = skip_spaces;
+				next_state = parse_name;
+			}
+			else if (*p == '@') {
+				seen_at = TRUE;
+			}
+
+			p++;
+			break;
+		case parse_quoted:
+			if (*p == '\\') {
+				if (p > c) {
+					g_string_append_len(ns, c, p - c);
+				}
+
+				p++;
+				c = p;
+			}
+			else if (*p == '"') {
+				if (p > c) {
+					g_string_append_len(ns, c, p - c);
+				}
+
+				if (p + 1 < end && g_ascii_isspace(p[1])) {
+					g_string_append_c(ns, ' ');
+				}
+
+				state = skip_spaces;
+				next_state = parse_name;
+			}
+			else if (*p == '@' && seen_obrace) {
+				seen_at = TRUE;
+			}
+			else if (*p == '<') {
+				seen_obrace = TRUE;
+			}
+			p++;
+			break;
+		case parse_addr:
+			if (*p == '>') {
+				int check = rspamd_email_address_check_and_add(c, p - c + 1,
+															   res, pool, ns, max_elements);
+				if (check == 0 && res->len == 0) {
+					/* Insert a fake address */
+					rspamd_email_address_add(pool, res, NULL, ns);
+				}
+				else if (check != 1) {
+					goto end;
+				}
+
+				/* Cleanup for the next use */
+				g_string_set_size(ns, 0);
+				seen_at = FALSE;
+				state = skip_spaces;
+				next_state = parse_name;
+			}
+			else if (*p == '@') {
+				seen_at = TRUE;
+			}
+			p++;
+			break;
+		case skip_spaces:
+			if (!g_ascii_isspace(*p)) {
+				c = p;
+				state = next_state;
+			}
+			else {
+				p++;
+			}
+			break;
+		}
+	}
+
+	/* Handle leftover */
+	switch (state) {
+	case parse_name:
+		/* Assume the whole header as name (bad thing) */
+		if (p > c) {
+			while (p > c && g_ascii_isspace(*p)) {
+				p--;
+			}
+
+			if (p > c) {
+				if (seen_at) {
+					/* The whole email is likely address */
+					int check = rspamd_email_address_check_and_add(c, p - c,
+																   res, pool, ns, max_elements);
+					if (check == 0 && res->len == 0) {
+						/* Insert a fake address */
+						rspamd_email_address_add(pool, res, NULL, ns);
+					}
+					else if (check != 1) {
+						goto end;
+					}
+				}
+				else {
+					/* No @ seen */
+					g_string_append_len(ns, c, p - c);
+
+					if (res->len == 0) {
+						rspamd_email_address_add(pool, res, NULL, ns);
+					}
+				}
+			}
+			else if (res->len == 0) {
+				rspamd_email_address_add(pool, res, NULL, ns);
+			}
+		}
+		break;
+	case parse_addr:
+		if (p > c) {
+			if (rspamd_email_address_check_and_add(c, p - c,
+												   res, pool, ns, max_elements) == 0) {
+				if (res->len == 0) {
+					rspamd_email_address_add(pool, res, NULL, ns);
+				}
+			}
+		}
+		break;
+	case parse_quoted:
+		/* Unfinished quoted string or a comment */
+		/* If we have seen obrace + at, then we still can try to resolve address */
+		if (seen_at && seen_obrace) {
+			p = rspamd_memrchr(cpy->str, '<', cpy->len);
+			g_assert(p != NULL);
+			if (rspamd_email_address_check_and_add(p, end - p,
+												   res, pool, ns, max_elements) == 0) {
+				if (res->len == 0) {
+					rspamd_email_address_add(pool, res, NULL, ns);
+				}
+			}
+		}
+		break;
+	default:
+		/* Do nothing */
+		break;
+	}
+end:
+	rspamd_mempool_notify_alloc(pool, cpy->len);
+	g_string_free(ns, TRUE);
+
+	return res;
+}
+
+void rspamd_email_address_list_destroy(gpointer ptr)
+{
+	GPtrArray *ar = ptr;
+	guint i;
+	struct rspamd_email_address *addr;
+
+	PTR_ARRAY_FOREACH(ar, i, addr)
+	{
+		rspamd_email_address_free(addr);
+	}
+
+	g_ptr_array_free(ar, TRUE);
+}
\ No newline at end of file
diff --git a/src/libmime/email_addr.h b/src/libmime/email_addr.h
new file mode 100644
index 0000000..ed00722
--- /dev/null
+++ b/src/libmime/email_addr.h
@@ -0,0 +1,97 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_EMAIL_ADDR_H_
+#define SRC_LIBMIME_EMAIL_ADDR_H_
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "libutil/ref.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_mime_header;
+
+enum rspamd_email_address_flags {
+	RSPAMD_EMAIL_ADDR_VALID = (1 << 0),
+	RSPAMD_EMAIL_ADDR_IP = (1 << 1),
+	RSPAMD_EMAIL_ADDR_BRACED = (1 << 2),
+	RSPAMD_EMAIL_ADDR_QUOTED = (1 << 3),
+	RSPAMD_EMAIL_ADDR_EMPTY = (1 << 4),
+	RSPAMD_EMAIL_ADDR_HAS_BACKSLASH = (1 << 5),
+	RSPAMD_EMAIL_ADDR_ADDR_ALLOCATED = (1 << 6),
+	RSPAMD_EMAIL_ADDR_USER_ALLOCATED = (1 << 7),
+	RSPAMD_EMAIL_ADDR_HAS_8BIT = (1 << 8),
+	RSPAMD_EMAIL_ADDR_ALIASED = (1 << 9),
+	RSPAMD_EMAIL_ADDR_ORIGINAL = (1 << 10),
+};
+
+/*
+ * Structure that represents email address in a convenient way
+ */
+struct rspamd_email_address {
+	const gchar *raw;
+	const gchar *addr;
+	const gchar *user;
+	const gchar *domain;
+	const gchar *name;
+
+	guint raw_len;
+	guint addr_len;
+	guint domain_len;
+	guint user_len;
+	guint flags;
+};
+
+struct rspamd_task;
+
+/**
+ * Create email address from a single rfc822 address (e.g. from mail from:)
+ * @param str string to use
+ * @param len length of string
+ * @return
+ */
+struct rspamd_email_address *rspamd_email_address_from_smtp(const gchar *str, guint len);
+
+/**
+ * Parses email address from the mime header, decodes names and return the array
+ * of `rspamd_email_address`. If `src` is NULL, then this function creates a new
+ * array and adds a destructor to remove elements when `pool` is destroyed.
+ * Otherwise, addresses are appended to `src`.
+ * @param hdr
+ * @param len
+ * @return
+ */
+GPtrArray *
+rspamd_email_address_from_mime(rspamd_mempool_t *pool, const gchar *hdr, guint len,
+							   GPtrArray *src, gint max_elements);
+
+/**
+ * Destroys list of email addresses
+ * @param ptr
+ */
+void rspamd_email_address_list_destroy(gpointer ptr);
+
+void rspamd_email_address_free(struct rspamd_email_address *addr);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_EMAIL_ADDR_H_ */
diff --git a/src/libmime/images.c b/src/libmime/images.c
new file mode 100644
index 0000000..1344d91
--- /dev/null
+++ b/src/libmime/images.c
@@ -0,0 +1,718 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "images.h"
+#include "task.h"
+#include "message.h"
+#include "libserver/html/html.h"
+
+#define msg_debug_images(...) rspamd_conditional_debug_fast(NULL, NULL,                                               \
+															rspamd_images_log_id, "images", task->task_pool->tag.uid, \
+															G_STRFUNC,                                                \
+															__VA_ARGS__)
+
+INIT_LOG_MODULE(images)
+
+#ifdef USABLE_GD
+#include "gd.h"
+#include "hash.h"
+#include <math.h>
+
+#define RSPAMD_NORMALIZED_DIM 64
+
+static rspamd_lru_hash_t *images_hash = NULL;
+#endif
+
+static const guint8 png_signature[] = {137, 80, 78, 71, 13, 10, 26, 10};
+static const guint8 jpg_sig1[] = {0xff, 0xd8};
+static const guint8 jpg_sig_jfif[] = {0xff, 0xe0};
+static const guint8 jpg_sig_exif[] = {0xff, 0xe1};
+static const guint8 gif_signature[] = {'G', 'I', 'F', '8'};
+static const guint8 bmp_signature[] = {'B', 'M'};
+
+static bool process_image(struct rspamd_task *task, struct rspamd_mime_part *part);
+
+
+bool rspamd_images_process_mime_part_maybe(struct rspamd_task *task,
+										   struct rspamd_mime_part *part)
+{
+	if (part->part_type == RSPAMD_MIME_PART_UNDEFINED) {
+		if (part->detected_type &&
+			strcmp(part->detected_type, "image") == 0 &&
+			part->parsed_data.len > 0) {
+
+			return process_image(task, part);
+		}
+	}
+
+	return false;
+}
+
+void rspamd_images_process(struct rspamd_task *task)
+{
+	guint i;
+	struct rspamd_mime_part *part;
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+	{
+		rspamd_images_process_mime_part_maybe(task, part);
+	}
+}
+
+static enum rspamd_image_type
+detect_image_type(rspamd_ftok_t *data)
+{
+	if (data->len > sizeof(png_signature) / sizeof(png_signature[0])) {
+		if (memcmp(data->begin, png_signature, sizeof(png_signature)) == 0) {
+			return IMAGE_TYPE_PNG;
+		}
+	}
+	if (data->len > 10) {
+		if (memcmp(data->begin, jpg_sig1, sizeof(jpg_sig1)) == 0) {
+			if (memcmp(data->begin + 2, jpg_sig_jfif, sizeof(jpg_sig_jfif)) == 0 ||
+				memcmp(data->begin + 2, jpg_sig_exif, sizeof(jpg_sig_exif)) == 0) {
+				return IMAGE_TYPE_JPG;
+			}
+		}
+	}
+	if (data->len > sizeof(gif_signature) / sizeof(gif_signature[0])) {
+		if (memcmp(data->begin, gif_signature, sizeof(gif_signature)) == 0) {
+			return IMAGE_TYPE_GIF;
+		}
+	}
+	if (data->len > sizeof(bmp_signature) / sizeof(bmp_signature[0])) {
+		if (memcmp(data->begin, bmp_signature, sizeof(bmp_signature)) == 0) {
+			return IMAGE_TYPE_BMP;
+		}
+	}
+
+	return IMAGE_TYPE_UNKNOWN;
+}
+
+
+static struct rspamd_image *
+process_png_image(rspamd_mempool_t *pool, rspamd_ftok_t *data)
+{
+	struct rspamd_image *img;
+	guint32 t;
+	const guint8 *p;
+
+	if (data->len < 24) {
+		msg_info_pool("bad png detected (maybe striped)");
+		return NULL;
+	}
+
+	/* In png we should find iHDR section and get data from it */
+	/* Skip signature and read header section */
+	p = data->begin + 12;
+	if (memcmp(p, "IHDR", 4) != 0) {
+		msg_info_pool("png doesn't begins with IHDR section");
+		return NULL;
+	}
+
+	img = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_image));
+	img->type = IMAGE_TYPE_PNG;
+	img->data = data;
+
+	p += 4;
+	memcpy(&t, p, sizeof(guint32));
+	img->width = ntohl(t);
+	p += 4;
+	memcpy(&t, p, sizeof(guint32));
+	img->height = ntohl(t);
+
+	return img;
+}
+
+static struct rspamd_image *
+process_jpg_image(rspamd_mempool_t *pool, rspamd_ftok_t *data)
+{
+	const guint8 *p, *end;
+	guint16 h, w;
+	struct rspamd_image *img;
+
+	img = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_image));
+	img->type = IMAGE_TYPE_JPG;
+	img->data = data;
+
+	p = data->begin;
+	end = p + data->len - 8;
+	p += 2;
+
+	while (p < end) {
+		if (p[0] == 0xFF && p[1] != 0xFF) {
+			guint len = p[2] * 256 + p[3];
+
+			p++;
+
+			if (*p == 0xc0 || *p == 0xc1 || *p == 0xc2 || *p == 0xc3 ||
+				*p == 0xc9 || *p == 0xca || *p == 0xcb) {
+				memcpy(&h, p + 4, sizeof(guint16));
+				h = p[4] * 0xff + p[5];
+				img->height = h;
+				w = p[6] * 0xff + p[7];
+				img->width = w;
+
+				return img;
+			}
+
+
+			p += len;
+		}
+		else {
+			p++;
+		}
+	}
+
+	return NULL;
+}
+
+static struct rspamd_image *
+process_gif_image(rspamd_mempool_t *pool, rspamd_ftok_t *data)
+{
+	struct rspamd_image *img;
+	const guint8 *p;
+	guint16 t;
+
+	if (data->len < 10) {
+		msg_info_pool("bad gif detected (maybe striped)");
+		return NULL;
+	}
+
+	img = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_image));
+	img->type = IMAGE_TYPE_GIF;
+	img->data = data;
+
+	p = data->begin + 6;
+	memcpy(&t, p, sizeof(guint16));
+	img->width = GUINT16_FROM_LE(t);
+	memcpy(&t, p + 2, sizeof(guint16));
+	img->height = GUINT16_FROM_LE(t);
+
+	return img;
+}
+
+static struct rspamd_image *
+process_bmp_image(rspamd_mempool_t *pool, rspamd_ftok_t *data)
+{
+	struct rspamd_image *img;
+	gint32 t;
+	const guint8 *p;
+
+	if (data->len < 28) {
+		msg_info_pool("bad bmp detected (maybe striped)");
+		return NULL;
+	}
+
+	img = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_image));
+	img->type = IMAGE_TYPE_BMP;
+	img->data = data;
+	p = data->begin + 18;
+	memcpy(&t, p, sizeof(guint32));
+	img->width = GUINT32_FROM_LE(t);
+	memcpy(&t, p + 4, sizeof(gint32));
+	img->height = GUINT32_FROM_LE(t);
+
+	return img;
+}
+
+#ifdef USABLE_GD
+/*
+ * DCT from Emil Mikulic.
+ * http://unix4lyfe.org/dct/
+ */
+static void
+rspamd_image_dct_block(gint pixels[8][8], gdouble *out)
+{
+	gint i;
+	gint rows[8][8];
+
+	static const gint c1 = 1004 /* cos(pi/16) << 10 */,
+					  s1 = 200 /* sin(pi/16) */,
+					  c3 = 851 /* cos(3pi/16) << 10 */,
+					  s3 = 569 /* sin(3pi/16) << 10 */,
+					  r2c6 = 554 /* sqrt(2)*cos(6pi/16) << 10 */,
+					  r2s6 = 1337 /* sqrt(2)*sin(6pi/16) << 10 */,
+					  r2 = 181; /* sqrt(2) << 7*/
+
+	gint x0, x1, x2, x3, x4, x5, x6, x7, x8;
+
+	/* transform rows */
+	for (i = 0; i < 8; i++) {
+		x0 = pixels[0][i];
+		x1 = pixels[1][i];
+		x2 = pixels[2][i];
+		x3 = pixels[3][i];
+		x4 = pixels[4][i];
+		x5 = pixels[5][i];
+		x6 = pixels[6][i];
+		x7 = pixels[7][i];
+
+		/* Stage 1 */
+		x8 = x7 + x0;
+		x0 -= x7;
+		x7 = x1 + x6;
+		x1 -= x6;
+		x6 = x2 + x5;
+		x2 -= x5;
+		x5 = x3 + x4;
+		x3 -= x4;
+
+		/* Stage 2 */
+		x4 = x8 + x5;
+		x8 -= x5;
+		x5 = x7 + x6;
+		x7 -= x6;
+		x6 = c1 * (x1 + x2);
+		x2 = (-s1 - c1) * x2 + x6;
+		x1 = (s1 - c1) * x1 + x6;
+		x6 = c3 * (x0 + x3);
+		x3 = (-s3 - c3) * x3 + x6;
+		x0 = (s3 - c3) * x0 + x6;
+
+		/* Stage 3 */
+		x6 = x4 + x5;
+		x4 -= x5;
+		x5 = r2c6 * (x7 + x8);
+		x7 = (-r2s6 - r2c6) * x7 + x5;
+		x8 = (r2s6 - r2c6) * x8 + x5;
+		x5 = x0 + x2;
+		x0 -= x2;
+		x2 = x3 + x1;
+		x3 -= x1;
+
+		/* Stage 4 and output */
+		rows[i][0] = x6;
+		rows[i][4] = x4;
+		rows[i][2] = x8 >> 10;
+		rows[i][6] = x7 >> 10;
+		rows[i][7] = (x2 - x5) >> 10;
+		rows[i][1] = (x2 + x5) >> 10;
+		rows[i][3] = (x3 * r2) >> 17;
+		rows[i][5] = (x0 * r2) >> 17;
+	}
+
+	/* transform columns */
+	for (i = 0; i < 8; i++) {
+		x0 = rows[0][i];
+		x1 = rows[1][i];
+		x2 = rows[2][i];
+		x3 = rows[3][i];
+		x4 = rows[4][i];
+		x5 = rows[5][i];
+		x6 = rows[6][i];
+		x7 = rows[7][i];
+
+		/* Stage 1 */
+		x8 = x7 + x0;
+		x0 -= x7;
+		x7 = x1 + x6;
+		x1 -= x6;
+		x6 = x2 + x5;
+		x2 -= x5;
+		x5 = x3 + x4;
+		x3 -= x4;
+
+		/* Stage 2 */
+		x4 = x8 + x5;
+		x8 -= x5;
+		x5 = x7 + x6;
+		x7 -= x6;
+		x6 = c1 * (x1 + x2);
+		x2 = (-s1 - c1) * x2 + x6;
+		x1 = (s1 - c1) * x1 + x6;
+		x6 = c3 * (x0 + x3);
+		x3 = (-s3 - c3) * x3 + x6;
+		x0 = (s3 - c3) * x0 + x6;
+
+		/* Stage 3 */
+		x6 = x4 + x5;
+		x4 -= x5;
+		x5 = r2c6 * (x7 + x8);
+		x7 = (-r2s6 - r2c6) * x7 + x5;
+		x8 = (r2s6 - r2c6) * x8 + x5;
+		x5 = x0 + x2;
+		x0 -= x2;
+		x2 = x3 + x1;
+		x3 -= x1;
+
+		/* Stage 4 and output */
+		out[i * 8] = (double) ((x6 + 16) >> 3);
+		out[i * 8 + 1] = (double) ((x4 + 16) >> 3);
+		out[i * 8 + 2] = (double) ((x8 + 16384) >> 13);
+		out[i * 8 + 3] = (double) ((x7 + 16384) >> 13);
+		out[i * 8 + 4] = (double) ((x2 - x5 + 16384) >> 13);
+		out[i * 8 + 5] = (double) ((x2 + x5 + 16384) >> 13);
+		out[i * 8 + 6] = (double) (((x3 >> 8) * r2 + 8192) >> 12);
+		out[i * 8 + 7] = (double) (((x0 >> 8) * r2 + 8192) >> 12);
+	}
+}
+
+struct rspamd_image_cache_entry {
+	guchar digest[64];
+	guchar dct[RSPAMD_DCT_LEN / NBBY];
+};
+
+static void
+rspamd_image_cache_entry_dtor(gpointer p)
+{
+	struct rspamd_image_cache_entry *entry = p;
+	g_free(entry);
+}
+
+static guint32
+rspamd_image_dct_hash(gconstpointer p)
+{
+	return rspamd_cryptobox_fast_hash(p, rspamd_cryptobox_HASHBYTES,
+									  rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_image_dct_equal(gconstpointer a, gconstpointer b)
+{
+	return memcmp(a, b, rspamd_cryptobox_HASHBYTES) == 0;
+}
+
+static void
+rspamd_image_create_cache(struct rspamd_config *cfg)
+{
+	images_hash = rspamd_lru_hash_new_full(cfg->images_cache_size, NULL,
+										   rspamd_image_cache_entry_dtor,
+										   rspamd_image_dct_hash, rspamd_image_dct_equal);
+}
+
+static gboolean
+rspamd_image_check_hash(struct rspamd_task *task, struct rspamd_image *img)
+{
+	struct rspamd_image_cache_entry *found;
+
+	if (images_hash == NULL) {
+		rspamd_image_create_cache(task->cfg);
+	}
+
+	found = rspamd_lru_hash_lookup(images_hash, img->parent->digest,
+								   task->tv.tv_sec);
+
+	if (found) {
+		/* We need to decompress */
+		img->dct = g_malloc(RSPAMD_DCT_LEN / NBBY);
+		rspamd_mempool_add_destructor(task->task_pool, g_free,
+									  img->dct);
+		/* Copy as found could be destroyed by LRU */
+		memcpy(img->dct, found->dct, RSPAMD_DCT_LEN / NBBY);
+		img->is_normalized = TRUE;
+
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static void
+rspamd_image_save_hash(struct rspamd_task *task, struct rspamd_image *img)
+{
+	struct rspamd_image_cache_entry *found;
+
+	if (img->is_normalized) {
+		found = rspamd_lru_hash_lookup(images_hash, img->parent->digest,
+									   task->tv.tv_sec);
+
+		if (!found) {
+			found = g_malloc0(sizeof(*found));
+			memcpy(found->dct, img->dct, RSPAMD_DCT_LEN / NBBY);
+			memcpy(found->digest, img->parent->digest, sizeof(found->digest));
+
+			rspamd_lru_hash_insert(images_hash, found->digest, found,
+								   task->tv.tv_sec, 0);
+		}
+	}
+}
+
+#endif
+
+void rspamd_image_normalize(struct rspamd_task *task, struct rspamd_image *img)
+{
+#ifdef USABLE_GD
+	gdImagePtr src = NULL, dst = NULL;
+	guint i, j, k, l;
+	gdouble *dct;
+
+	if (img->data->len == 0 || img->data->len > G_MAXINT32) {
+		return;
+	}
+
+	if (img->height <= RSPAMD_NORMALIZED_DIM ||
+		img->width <= RSPAMD_NORMALIZED_DIM) {
+		return;
+	}
+
+	if (img->data->len > task->cfg->max_pic_size) {
+		return;
+	}
+
+	if (rspamd_image_check_hash(task, img)) {
+		return;
+	}
+
+	switch (img->type) {
+	case IMAGE_TYPE_JPG:
+		src = gdImageCreateFromJpegPtr(img->data->len, (void *) img->data->begin);
+		break;
+	case IMAGE_TYPE_PNG:
+		src = gdImageCreateFromPngPtr(img->data->len, (void *) img->data->begin);
+		break;
+	case IMAGE_TYPE_GIF:
+		src = gdImageCreateFromGifPtr(img->data->len, (void *) img->data->begin);
+		break;
+	case IMAGE_TYPE_BMP:
+		src = gdImageCreateFromBmpPtr(img->data->len, (void *) img->data->begin);
+		break;
+	default:
+		return;
+	}
+
+	if (src == NULL) {
+		msg_info_task("cannot load image of type %s from %T",
+					  rspamd_image_type_str(img->type), img->filename);
+	}
+	else {
+		gdImageSetInterpolationMethod(src, GD_BILINEAR_FIXED);
+
+		dst = gdImageScale(src, RSPAMD_NORMALIZED_DIM, RSPAMD_NORMALIZED_DIM);
+		gdImageGrayScale(dst);
+		gdImageDestroy(src);
+
+		img->is_normalized = TRUE;
+		dct = g_malloc0(sizeof(gdouble) * RSPAMD_DCT_LEN);
+		img->dct = g_malloc0(RSPAMD_DCT_LEN / NBBY);
+		rspamd_mempool_add_destructor(task->task_pool, g_free,
+									  img->dct);
+
+		/*
+		 * Split message into blocks:
+		 *
+		 * ****
+		 * ****
+		 *
+		 * Get sum of saturation values, and set bit if sum is > avg
+		 * Then go further
+		 *
+		 * ****
+		 * ****
+		 *
+		 * and repeat this algorithm.
+		 *
+		 * So on each iteration we move by 16 pixels and calculate 2 elements of
+		 * signature
+		 */
+		for (i = 0; i < RSPAMD_NORMALIZED_DIM; i += 8) {
+			for (j = 0; j < RSPAMD_NORMALIZED_DIM; j += 8) {
+				gint p[8][8];
+
+				for (k = 0; k < 8; k++) {
+					p[k][0] = gdImageGetPixel(dst, i + k, j);
+					p[k][1] = gdImageGetPixel(dst, i + k, j + 1);
+					p[k][2] = gdImageGetPixel(dst, i + k, j + 2);
+					p[k][3] = gdImageGetPixel(dst, i + k, j + 3);
+					p[k][4] = gdImageGetPixel(dst, i + k, j + 4);
+					p[k][5] = gdImageGetPixel(dst, i + k, j + 5);
+					p[k][6] = gdImageGetPixel(dst, i + k, j + 6);
+					p[k][7] = gdImageGetPixel(dst, i + k, j + 7);
+				}
+
+				rspamd_image_dct_block(p,
+									   dct + i * RSPAMD_NORMALIZED_DIM + j);
+
+				gdouble avg = 0.0;
+
+				for (k = 0; k < 8; k++) {
+					for (l = 0; l < 8; l++) {
+						gdouble x = *(dct +
+									  i * RSPAMD_NORMALIZED_DIM + j + k * 8 + l);
+						avg += (x - avg) / (gdouble) (k * 8 + l + 1);
+					}
+				}
+
+
+				for (k = 0; k < 8; k++) {
+					for (l = 0; l < 8; l++) {
+						guint idx = i * RSPAMD_NORMALIZED_DIM + j + k * 8 + l;
+
+						if (dct[idx] >= avg) {
+							setbit(img->dct, idx);
+						}
+					}
+				}
+			}
+		}
+
+		gdImageDestroy(dst);
+		g_free(dct);
+		rspamd_image_save_hash(task, img);
+	}
+#endif
+}
+
+struct rspamd_image *
+rspamd_maybe_process_image(rspamd_mempool_t *pool,
+						   rspamd_ftok_t *data)
+{
+	enum rspamd_image_type type;
+	struct rspamd_image *img = NULL;
+
+	if ((type = detect_image_type(data)) != IMAGE_TYPE_UNKNOWN) {
+		switch (type) {
+		case IMAGE_TYPE_PNG:
+			img = process_png_image(pool, data);
+			break;
+		case IMAGE_TYPE_JPG:
+			img = process_jpg_image(pool, data);
+			break;
+		case IMAGE_TYPE_GIF:
+			img = process_gif_image(pool, data);
+			break;
+		case IMAGE_TYPE_BMP:
+			img = process_bmp_image(pool, data);
+			break;
+		default:
+			img = NULL;
+			break;
+		}
+	}
+
+	return img;
+}
+
+static bool
+process_image(struct rspamd_task *task, struct rspamd_mime_part *part)
+{
+	struct rspamd_image *img;
+
+	img = rspamd_maybe_process_image(task->task_pool, &part->parsed_data);
+
+	if (img != NULL) {
+		msg_debug_images("detected %s image of size %ud x %ud",
+						 rspamd_image_type_str(img->type),
+						 img->width, img->height);
+
+		if (part->cd) {
+			img->filename = &part->cd->filename;
+		}
+
+		img->parent = part;
+
+		part->part_type = RSPAMD_MIME_PART_IMAGE;
+		part->specific.img = img;
+
+		return true;
+	}
+
+	return false;
+}
+
+const gchar *
+rspamd_image_type_str(enum rspamd_image_type type)
+{
+	switch (type) {
+	case IMAGE_TYPE_PNG:
+		return "PNG";
+		break;
+	case IMAGE_TYPE_JPG:
+		return "JPEG";
+		break;
+	case IMAGE_TYPE_GIF:
+		return "GIF";
+		break;
+	case IMAGE_TYPE_BMP:
+		return "BMP";
+		break;
+	default:
+		break;
+	}
+
+	return "unknown";
+}
+
+static void
+rspamd_image_process_part(struct rspamd_task *task, struct rspamd_mime_part *part)
+{
+	struct rspamd_mime_header *rh;
+	struct rspamd_mime_text_part *tp;
+	struct html_image *himg;
+	const gchar *cid;
+	guint cid_len, i;
+	struct rspamd_image *img;
+
+	img = (struct rspamd_image *) part->specific.img;
+
+	if (img) {
+		/* Check Content-Id */
+		rh = rspamd_message_get_header_from_hash(part->raw_headers,
+												 "Content-Id", FALSE);
+
+		if (rh) {
+			cid = rh->decoded;
+
+			if (*cid == '<') {
+				cid++;
+			}
+
+			cid_len = strlen(cid);
+
+			if (cid_len > 0) {
+				if (cid[cid_len - 1] == '>') {
+					cid_len--;
+				}
+
+				PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, tp)
+				{
+					if (IS_TEXT_PART_HTML(tp) && tp->html != NULL) {
+						himg = rspamd_html_find_embedded_image(tp->html, cid, cid_len);
+
+						if (himg != NULL) {
+							img->html_image = himg;
+							himg->embedded_image = img;
+
+							msg_debug_images("found linked image by cid: <%s>",
+											 cid);
+
+							if (himg->height == 0) {
+								himg->height = img->height;
+							}
+
+							if (himg->width == 0) {
+								himg->width = img->width;
+							}
+						}
+					}
+				}
+			}
+		}
+	}
+}
+
+void rspamd_images_link(struct rspamd_task *task)
+{
+	struct rspamd_mime_part *part;
+	guint i;
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+	{
+		if (part->part_type == RSPAMD_MIME_PART_IMAGE) {
+			rspamd_image_process_part(task, part);
+		}
+	}
+}
\ No newline at end of file
diff --git a/src/libmime/images.h b/src/libmime/images.h
new file mode 100644
index 0000000..bf8b3be
--- /dev/null
+++ b/src/libmime/images.h
@@ -0,0 +1,76 @@
+#ifndef IMAGES_H_
+#define IMAGES_H_
+
+#include "config.h"
+#include "fstring.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct html_image;
+struct rspamd_task;
+struct rspamd_mime_part;
+
+#define RSPAMD_DCT_LEN (64 * 64)
+
+enum rspamd_image_type {
+	IMAGE_TYPE_PNG = 0,
+	IMAGE_TYPE_JPG,
+	IMAGE_TYPE_GIF,
+	IMAGE_TYPE_BMP,
+	IMAGE_TYPE_UNKNOWN
+};
+
+struct rspamd_image {
+	struct rspamd_mime_part *parent;
+	rspamd_ftok_t *data;
+	rspamd_ftok_t *filename;
+	struct html_image *html_image;
+	enum rspamd_image_type type;
+	guint32 width;
+	guint32 height;
+	gboolean is_normalized;
+	guchar *dct;
+};
+
+/*
+ * Process images from a worker task
+ */
+void rspamd_images_process(struct rspamd_task *task);
+
+/**
+ * Process image if possible in a single mime part
+ * @param task
+ * @param part
+ * @return
+ */
+bool rspamd_images_process_mime_part_maybe(struct rspamd_task *task,
+										   struct rspamd_mime_part *part);
+
+/*
+ * Link embedded images to the HTML parts
+ */
+void rspamd_images_link(struct rspamd_task *task);
+
+/**
+ * Processes image in raw data
+ * @param task
+ * @param data
+ * @return
+ */
+struct rspamd_image *rspamd_maybe_process_image(rspamd_mempool_t *pool,
+												rspamd_ftok_t *data);
+
+/*
+ * Get textual representation of an image's type
+ */
+const gchar *rspamd_image_type_str(enum rspamd_image_type type);
+
+void rspamd_image_normalize(struct rspamd_task *task, struct rspamd_image *img);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* IMAGES_H_ */
diff --git a/src/libmime/lang_detection.c b/src/libmime/lang_detection.c
new file mode 100644
index 0000000..bdd0aad
--- /dev/null
+++ b/src/libmime/lang_detection.c
@@ -0,0 +1,2103 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lang_detection.h"
+#include "lang_detection_fasttext.h"
+#include "libserver/logger.h"
+#include "libcryptobox/cryptobox.h"
+#include "libutil/multipattern.h"
+#include "ucl.h"
+#include "khash.h"
+#include "libstemmer.h"
+
+#include <glob.h>
+#include <unicode/utf8.h>
+#include <unicode/utf16.h>
+#include <unicode/ucnv.h>
+#include <unicode/uchar.h>
+#include <unicode/ustring.h>
+#include <math.h>
+
+static const gsize default_short_text_limit = 10;
+static const gsize default_words = 80;
+static const gdouble update_prob = 0.6;
+static const gchar *default_languages_path = RSPAMD_SHAREDIR "/languages";
+
+#undef EXTRA_LANGDET_DEBUG
+
+struct rspamd_language_unicode_match {
+	const gchar *lang;
+	gint unicode_code;
+};
+
+/*
+ * List of languages detected by unicode scripts
+ */
+static const struct rspamd_language_unicode_match unicode_langs[] = {
+	{"el", RSPAMD_UNICODE_GREEK},
+	{"ml", RSPAMD_UNICODE_MALAYALAM},
+	{"te", RSPAMD_UNICODE_TELUGU},
+	{"ta", RSPAMD_UNICODE_TAMIL},
+	{"gu", RSPAMD_UNICODE_GUJARATI},
+	{"th", RSPAMD_UNICODE_THAI},
+	{"ka", RSPAMD_UNICODE_GEORGIAN},
+	{"si", RSPAMD_UNICODE_SINHALA},
+	{"hy", RSPAMD_UNICODE_ARMENIAN},
+	{"ja", RSPAMD_UNICODE_JP},
+	{"ko", RSPAMD_UNICODE_HANGUL},
+};
+
+/*
+ * Top languages
+ */
+static const gchar *tier0_langs[] = {
+	"en",
+};
+static const gchar *tier1_langs[] = {
+	"fr", "it", "de", "es", "nl",
+	"pt", "ru", "pl", "tk", "th", "ar"};
+
+enum rspamd_language_category {
+	RSPAMD_LANGUAGE_LATIN = 0,
+	RSPAMD_LANGUAGE_CYRILLIC,
+	RSPAMD_LANGUAGE_DEVANAGARI,
+	RSPAMD_LANGUAGE_ARAB,
+	RSPAMD_LANGUAGE_MAX,
+};
+
+struct rspamd_language_elt {
+	const gchar *name; /* e.g. "en" or "ru" */
+	gint flags;        /* enum rspamd_language_elt_flags */
+	enum rspamd_language_category category;
+	guint trigrams_words;
+	guint stop_words;
+	gdouble mean;
+	gdouble std;
+	guint occurrences; /* total number of parts with this language */
+};
+
+struct rspamd_ngramm_elt {
+	struct rspamd_language_elt *elt;
+	gdouble prob;
+};
+
+struct rspamd_ngramm_chain {
+	GPtrArray *languages;
+	gdouble mean;
+	gdouble std;
+	gchar *utf;
+};
+
+struct rspamd_stop_word_range {
+	guint start;
+	guint stop;
+	struct rspamd_language_elt *elt;
+};
+
+struct rspamd_stop_word_elt {
+	struct rspamd_multipattern *mp;
+	GArray *ranges; /* of rspamd_stop_word_range */
+};
+
+#define msg_debug_lang_det(...) rspamd_conditional_debug_fast(NULL, NULL,                                                 \
+															  rspamd_langdet_log_id, "langdet", task->task_pool->tag.uid, \
+															  G_STRFUNC,                                                  \
+															  __VA_ARGS__)
+#define msg_debug_lang_det_cfg(...) rspamd_conditional_debug_fast(NULL, NULL,                                               \
+																  rspamd_langdet_log_id, "langdet", cfg->cfg_pool->tag.uid, \
+																  G_STRFUNC,                                                \
+																  __VA_ARGS__)
+
+INIT_LOG_MODULE_PUBLIC(langdet)
+
+static const struct rspamd_language_unicode_match *
+rspamd_language_search_unicode_match(const gchar *key,
+									 const struct rspamd_language_unicode_match *elts, size_t nelts)
+{
+	size_t i;
+
+	for (i = 0; i < nelts; i++) {
+		if (strcmp(elts[i].lang, key) == 0) {
+			return &elts[i];
+		}
+	}
+
+	return NULL;
+}
+
+static gboolean
+rspamd_language_search_str(const gchar *key, const gchar *elts[], size_t nelts)
+{
+	size_t i;
+
+	for (i = 0; i < nelts; i++) {
+		if (strcmp(elts[i], key) == 0) {
+			return TRUE;
+		}
+	}
+	return FALSE;
+}
+
+static guint
+rspamd_trigram_hash_func(gconstpointer key)
+{
+	return rspamd_cryptobox_fast_hash(key, 3 * sizeof(UChar32),
+									  rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_trigram_equal_func(gconstpointer v, gconstpointer v2)
+{
+	return memcmp(v, v2, 3 * sizeof(UChar32)) == 0;
+}
+
+KHASH_INIT(rspamd_trigram_hash, const UChar32 *, struct rspamd_ngramm_chain, true,
+		   rspamd_trigram_hash_func, rspamd_trigram_equal_func);
+KHASH_INIT(rspamd_candidates_hash, const gchar *,
+		   struct rspamd_lang_detector_res *, true,
+		   rspamd_str_hash, rspamd_str_equal);
+KHASH_INIT(rspamd_stopwords_hash, rspamd_ftok_t *,
+		   char, false,
+		   rspamd_ftok_hash, rspamd_ftok_equal);
+
+KHASH_INIT(rspamd_languages_hash, const gchar *, struct rspamd_language_elt *, true,
+		   rspamd_str_hash, rspamd_str_equal);
+struct rspamd_lang_detector {
+	khash_t(rspamd_languages_hash) * languages;
+	khash_t(rspamd_trigram_hash) * trigrams[RSPAMD_LANGUAGE_MAX]; /* trigrams frequencies */
+	struct rspamd_stop_word_elt stop_words[RSPAMD_LANGUAGE_MAX];
+	khash_t(rspamd_stopwords_hash) * stop_words_norm;
+	UConverter *uchar_converter;
+	gsize short_text_limit;
+	bool prefer_fasttext;
+	gsize total_occurrences; /* number of all languages found */
+	gpointer fasttext_detector;
+	ref_entry_t ref;
+};
+
+static void
+rspamd_language_detector_ucs_lowercase(UChar32 *s, gsize len)
+{
+	gsize i;
+
+	for (i = 0; i < len; i++) {
+		s[i] = u_tolower(s[i]);
+	}
+}
+
+static gboolean
+rspamd_language_detector_ucs_is_latin(const UChar32 *s, gsize len)
+{
+	gsize i;
+	gboolean ret = TRUE;
+
+	for (i = 0; i < len; i++) {
+		if (s[i] >= 128 || !(g_ascii_isalnum(s[i]) || s[i] == ' ')) {
+			ret = FALSE;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+struct rspamd_language_ucs_elt {
+	guint freq;
+	const gchar *utf;
+	UChar32 s[0];
+};
+
+static void
+rspamd_language_detector_init_ngramm(struct rspamd_config *cfg,
+									 struct rspamd_lang_detector *d,
+									 struct rspamd_language_elt *lelt,
+									 struct rspamd_language_ucs_elt *ucs,
+									 guint len,
+									 guint freq,
+									 guint total,
+									 khash_t(rspamd_trigram_hash) * htb)
+{
+	struct rspamd_ngramm_chain *chain = NULL, st_chain;
+	struct rspamd_ngramm_elt *elt;
+	khiter_t k;
+	guint i;
+	gboolean found;
+
+	switch (len) {
+	case 1:
+	case 2:
+		g_assert_not_reached();
+		break;
+	case 3:
+		k = kh_get(rspamd_trigram_hash, htb, ucs->s);
+		if (k != kh_end(htb)) {
+			chain = &kh_value(htb, k);
+		}
+		break;
+	default:
+		g_assert_not_reached();
+		break;
+	}
+
+	if (chain == NULL) {
+		/* New element */
+		chain = &st_chain;
+		memset(chain, 0, sizeof(st_chain));
+		chain->languages = g_ptr_array_sized_new(32);
+		rspamd_mempool_add_destructor(cfg->cfg_pool, rspamd_ptr_array_free_hard,
+									  chain->languages);
+		chain->utf = rspamd_mempool_strdup(cfg->cfg_pool, ucs->utf);
+		elt = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(*elt));
+		elt->elt = lelt;
+		elt->prob = ((gdouble) freq) / ((gdouble) total);
+		g_ptr_array_add(chain->languages, elt);
+
+		k = kh_put(rspamd_trigram_hash, htb, ucs->s, &i);
+		kh_value(htb, k) = *chain;
+	}
+	else {
+		/* Check sanity */
+		found = FALSE;
+
+		PTR_ARRAY_FOREACH(chain->languages, i, elt)
+		{
+			if (strcmp(elt->elt->name, lelt->name) == 0) {
+				found = TRUE;
+				elt->prob += ((gdouble) freq) / ((gdouble) total);
+				break;
+			}
+		}
+
+		if (!found) {
+			elt = rspamd_mempool_alloc(cfg->cfg_pool, sizeof(*elt));
+			elt->elt = lelt;
+			elt->prob = ((gdouble) freq) / ((gdouble) total);
+			g_ptr_array_add(chain->languages, elt);
+		}
+	}
+}
+
+static inline enum rspamd_language_category
+rspamd_language_detector_get_category(guint uflags)
+{
+	enum rspamd_language_category cat = RSPAMD_LANGUAGE_LATIN;
+
+	if (uflags & RSPAMD_UNICODE_CYRILLIC) {
+		cat = RSPAMD_LANGUAGE_CYRILLIC;
+	}
+	else if (uflags & RSPAMD_UNICODE_DEVANAGARI) {
+		cat = RSPAMD_LANGUAGE_DEVANAGARI;
+	}
+	else if (uflags & RSPAMD_UNICODE_ARABIC) {
+		cat = RSPAMD_LANGUAGE_ARAB;
+	}
+
+	return cat;
+}
+
+static const gchar *
+rspamd_language_detector_print_flags(struct rspamd_language_elt *elt)
+{
+	static gchar flags_buf[256];
+	goffset r = 0;
+
+	if (elt->flags & RS_LANGUAGE_TIER1) {
+		r += rspamd_snprintf(flags_buf + r, sizeof(flags_buf) - r, "tier1,");
+	}
+	if (elt->flags & RS_LANGUAGE_TIER0) {
+		r += rspamd_snprintf(flags_buf + r, sizeof(flags_buf) - r, "tier0,");
+	}
+	if (elt->flags & RS_LANGUAGE_LATIN) {
+		r += rspamd_snprintf(flags_buf + r, sizeof(flags_buf) - r, "latin,");
+	}
+
+	if (r > 0) {
+		flags_buf[r - 1] = '\0';
+	}
+	else {
+		flags_buf[r] = '\0';
+	}
+
+	return flags_buf;
+}
+
+static gint
+rspamd_language_detector_cmp_ngramm(gconstpointer a, gconstpointer b)
+{
+	struct rspamd_language_ucs_elt *e1 = *(struct rspamd_language_ucs_elt **) a;
+	struct rspamd_language_ucs_elt *e2 = *(struct rspamd_language_ucs_elt **) b;
+
+	return (gint) e2->freq - (gint) e1->freq;
+}
+
+static void
+rspamd_language_detector_read_file(struct rspamd_config *cfg,
+								   struct rspamd_lang_detector *d,
+								   const gchar *path,
+								   const ucl_object_t *stop_words)
+{
+	struct ucl_parser *parser;
+	ucl_object_t *top;
+	const ucl_object_t *freqs, *n_words, *cur, *type, *flags;
+	ucl_object_iter_t it = NULL;
+	UErrorCode uc_err = U_ZERO_ERROR;
+	struct rspamd_language_elt *nelt;
+	struct rspamd_language_ucs_elt *ucs_elt;
+	khash_t(rspamd_trigram_hash) *htb = NULL;
+	gchar *pos;
+	guint total = 0, total_latin = 0, total_ngramms = 0, i, skipped,
+		  loaded;
+	gdouble mean = 0, std = 0, delta = 0, delta2 = 0, m2 = 0;
+	enum rspamd_language_category cat = RSPAMD_LANGUAGE_MAX;
+
+	parser = ucl_parser_new(UCL_PARSER_NO_FILEVARS);
+	if (!ucl_parser_add_file(parser, path)) {
+		msg_warn_config("cannot parse file %s: %s", path,
+						ucl_parser_get_error(parser));
+		ucl_parser_free(parser);
+
+		return;
+	}
+
+	top = ucl_parser_get_object(parser);
+	ucl_parser_free(parser);
+
+	freqs = ucl_object_lookup(top, "freq");
+
+	if (freqs == NULL) {
+		msg_warn_config("file %s has no 'freq' key", path);
+		ucl_object_unref(top);
+
+		return;
+	}
+
+	pos = strrchr(path, '/');
+	g_assert(pos != NULL);
+	nelt = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*nelt));
+	nelt->name = rspamd_mempool_strdup(cfg->cfg_pool, pos + 1);
+	/* Remove extension */
+	pos = strchr(nelt->name, '.');
+	g_assert(pos != NULL);
+	*pos = '\0';
+
+	n_words = ucl_object_lookup(top, "n_words");
+
+	if (n_words == NULL || ucl_object_type(n_words) != UCL_ARRAY ||
+		n_words->len != 3) {
+		msg_warn_config("cannot find n_words in language %s", nelt->name);
+		ucl_object_unref(top);
+
+		return;
+	}
+	else {
+		nelt->trigrams_words = ucl_object_toint(ucl_array_find_index(n_words,
+																	 2));
+	}
+
+	type = ucl_object_lookup(top, "type");
+
+	if (type == NULL || ucl_object_type(type) != UCL_STRING) {
+		msg_debug_config("cannot find type in language %s", nelt->name);
+		ucl_object_unref(top);
+
+		return;
+	}
+	else {
+		const gchar *stype = ucl_object_tostring(type);
+
+		if (strcmp(stype, "latin") == 0) {
+			cat = RSPAMD_LANGUAGE_LATIN;
+		}
+		else if (strcmp(stype, "cyrillic") == 0) {
+			cat = RSPAMD_LANGUAGE_CYRILLIC;
+		}
+		else if (strcmp(stype, "arab") == 0) {
+			cat = RSPAMD_LANGUAGE_ARAB;
+		}
+		else if (strcmp(stype, "devanagari") == 0) {
+			cat = RSPAMD_LANGUAGE_DEVANAGARI;
+		}
+		else {
+			msg_debug_config("unknown type %s of language %s", stype, nelt->name);
+			ucl_object_unref(top);
+
+			return;
+		}
+	}
+
+	flags = ucl_object_lookup(top, "flags");
+
+	if (flags != NULL && ucl_object_type(flags) == UCL_ARRAY) {
+		ucl_object_iter_t it = NULL;
+		const ucl_object_t *cur;
+
+		while ((cur = ucl_object_iterate(flags, &it, true)) != NULL) {
+			const gchar *fl = ucl_object_tostring(cur);
+
+			if (cur) {
+				if (strcmp(fl, "diacritics") == 0) {
+					nelt->flags |= RS_LANGUAGE_DIACRITICS;
+				}
+				else if (strcmp(fl, "ascii") == 0) {
+					nelt->flags |= RS_LANGUAGE_ASCII;
+				}
+				else {
+					msg_debug_config("unknown flag %s of language %s", fl, nelt->name);
+				}
+			}
+			else {
+				msg_debug_config("unknown flags type of language %s", nelt->name);
+			}
+		}
+	}
+
+	if (stop_words) {
+		const ucl_object_t *specific_stop_words;
+
+		specific_stop_words = ucl_object_lookup(stop_words, nelt->name);
+
+		if (specific_stop_words) {
+			struct sb_stemmer *stem = NULL;
+			it = NULL;
+			const ucl_object_t *w;
+			guint start, stop;
+
+			stem = sb_stemmer_new(nelt->name, "UTF_8");
+			start = rspamd_multipattern_get_npatterns(d->stop_words[cat].mp);
+
+			while ((w = ucl_object_iterate(specific_stop_words, &it, true)) != NULL) {
+				gsize wlen;
+				const char *word = ucl_object_tolstring(w, &wlen);
+				const char *saved;
+				guint mp_flags = RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8;
+
+				if (rspamd_multipattern_has_hyperscan()) {
+					mp_flags |= RSPAMD_MULTIPATTERN_RE;
+				}
+
+				rspamd_multipattern_add_pattern_len(d->stop_words[cat].mp,
+													word, wlen,
+													mp_flags);
+				nelt->stop_words++;
+
+				/* Also lemmatise and store normalised */
+				if (stem) {
+					const char *nw = sb_stemmer_stem(stem, word, wlen);
+
+
+					if (nw) {
+						saved = nw;
+						wlen = strlen(nw);
+					}
+					else {
+						saved = word;
+					}
+				}
+				else {
+					saved = word;
+				}
+
+				if (saved) {
+					gint rc;
+					rspamd_ftok_t *tok;
+					gchar *dst;
+
+					tok = rspamd_mempool_alloc(cfg->cfg_pool,
+											   sizeof(*tok) + wlen + 1);
+					dst = ((gchar *) tok) + sizeof(*tok);
+					rspamd_strlcpy(dst, saved, wlen + 1);
+					tok->begin = dst;
+					tok->len = wlen;
+
+					kh_put(rspamd_stopwords_hash, d->stop_words_norm,
+						   tok, &rc);
+				}
+			}
+
+			if (stem) {
+				sb_stemmer_delete(stem);
+			}
+
+			stop = rspamd_multipattern_get_npatterns(d->stop_words[cat].mp);
+
+			struct rspamd_stop_word_range r;
+
+			r.start = start;
+			r.stop = stop;
+			r.elt = nelt;
+
+			g_array_append_val(d->stop_words[cat].ranges, r);
+			it = NULL;
+		}
+	}
+
+	nelt->category = cat;
+	htb = d->trigrams[cat];
+
+	GPtrArray *ngramms;
+	guint nsym;
+
+	if (rspamd_language_search_str(nelt->name, tier1_langs,
+								   G_N_ELEMENTS(tier1_langs))) {
+		nelt->flags |= RS_LANGUAGE_TIER1;
+	}
+
+	if (rspamd_language_search_str(nelt->name, tier0_langs,
+								   G_N_ELEMENTS(tier0_langs))) {
+		nelt->flags |= RS_LANGUAGE_TIER0;
+	}
+
+	it = NULL;
+	ngramms = g_ptr_array_sized_new(freqs->len);
+	i = 0;
+	skipped = 0;
+	loaded = 0;
+
+	while ((cur = ucl_object_iterate(freqs, &it, true)) != NULL) {
+		const gchar *key;
+		gsize keylen;
+		guint freq;
+
+		key = ucl_object_keyl(cur, &keylen);
+		freq = ucl_object_toint(cur);
+
+		i++;
+		delta = freq - mean;
+		mean += delta / i;
+		delta2 = freq - mean;
+		m2 += delta * delta2;
+
+		if (key != NULL) {
+			UChar32 *cur_ucs;
+			const char *end = key + keylen, *cur_utf = key;
+
+			ucs_elt = rspamd_mempool_alloc(cfg->cfg_pool,
+										   sizeof(*ucs_elt) + (keylen + 1) * sizeof(UChar32));
+
+			cur_ucs = ucs_elt->s;
+			nsym = 0;
+			uc_err = U_ZERO_ERROR;
+
+			while (cur_utf < end) {
+				*cur_ucs++ = ucnv_getNextUChar(d->uchar_converter, &cur_utf,
+											   end, &uc_err);
+				if (!U_SUCCESS(uc_err)) {
+					break;
+				}
+
+				nsym++;
+			}
+
+			if (!U_SUCCESS(uc_err)) {
+				msg_warn_config("cannot convert key %*s to unicode: %s",
+								(gint) keylen, key, u_errorName(uc_err));
+
+				continue;
+			}
+
+			ucs_elt->utf = key;
+			rspamd_language_detector_ucs_lowercase(ucs_elt->s, nsym);
+
+			if (nsym == 3) {
+				g_ptr_array_add(ngramms, ucs_elt);
+			}
+			else {
+				continue;
+			}
+
+			if (rspamd_language_detector_ucs_is_latin(ucs_elt->s, nsym)) {
+				total_latin++;
+			}
+
+			ucs_elt->freq = freq;
+
+			total_ngramms++;
+		}
+	}
+
+	std = sqrt(m2 / (i - 1));
+
+	if (total_latin >= total_ngramms / 3) {
+		nelt->flags |= RS_LANGUAGE_LATIN;
+	}
+
+	nsym = 3;
+
+	total = 0;
+	PTR_ARRAY_FOREACH(ngramms, i, ucs_elt)
+	{
+
+		if (!(nelt->flags & RS_LANGUAGE_LATIN) &&
+			rspamd_language_detector_ucs_is_latin(ucs_elt->s, nsym)) {
+			ucs_elt->freq = 0;
+			/* Skip latin ngramm for non-latin language to avoid garbage */
+			skipped++;
+			continue;
+		}
+
+		/* Now, discriminate low frequency ngramms */
+
+		total += ucs_elt->freq;
+		loaded++;
+	}
+
+	g_ptr_array_sort(ngramms, rspamd_language_detector_cmp_ngramm);
+
+	PTR_ARRAY_FOREACH(ngramms, i, ucs_elt)
+	{
+		if (ucs_elt->freq > 0) {
+			rspamd_language_detector_init_ngramm(cfg, d,
+												 nelt, ucs_elt, nsym,
+												 ucs_elt->freq, total, htb);
+		}
+	}
+
+#ifdef EXTRA_LANGDET_DEBUG
+	/* Useful for debug */
+	for (i = 0; i < 10; i++) {
+		ucs_elt = g_ptr_array_index(ngramms, i);
+
+		msg_debug_lang_det_cfg("%s -> %s: %d", nelt->name,
+							   ucs_elt->utf, ucs_elt->freq);
+	}
+#endif
+
+	g_ptr_array_free(ngramms, TRUE);
+	nelt->mean = mean;
+	nelt->std = std;
+
+	msg_debug_lang_det_cfg("loaded %s language, %d trigrams, "
+						   "%d ngramms loaded; "
+						   "std=%.2f, mean=%.2f, skipped=%d, loaded=%d, stop_words=%d; "
+						   "(%s)",
+						   nelt->name,
+						   (gint) nelt->trigrams_words,
+						   total,
+						   std, mean,
+						   skipped, loaded, nelt->stop_words,
+						   rspamd_language_detector_print_flags(nelt));
+
+	int ret;
+	khiter_t k = kh_put(rspamd_languages_hash, d->languages, nelt->name, &ret);
+	g_assert(ret > 0); /* must be unique */
+	kh_value(d->languages, k) = nelt;
+	ucl_object_unref(top);
+}
+
+static gboolean
+rspamd_ucl_array_find_str(const gchar *str, const ucl_object_t *ar)
+{
+	ucl_object_iter_t it = NULL;
+	const ucl_object_t *cur;
+
+	if (ar == NULL || ar->len == 0) {
+		return FALSE;
+	}
+
+	while ((cur = ucl_object_iterate(ar, &it, true)) != NULL) {
+		if (ucl_object_type(cur) == UCL_STRING && rspamd_strcase_equal(
+													  ucl_object_tostring(cur), str)) {
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+static void
+rspamd_language_detector_process_chain(struct rspamd_config *cfg,
+									   struct rspamd_ngramm_chain *chain)
+{
+	struct rspamd_ngramm_elt *elt;
+	guint i;
+	gdouble delta, mean = 0, delta2, m2 = 0, std;
+
+	if (chain->languages->len > 3) {
+		PTR_ARRAY_FOREACH(chain->languages, i, elt)
+		{
+			delta = elt->prob - mean;
+			mean += delta / (i + 1);
+			delta2 = elt->prob - mean;
+			m2 += delta * delta2;
+		}
+
+		std = sqrt(m2 / (i - 1));
+		chain->mean = mean;
+		chain->std = std;
+
+		/* Now, filter elements that are lower than mean */
+		PTR_ARRAY_FOREACH(chain->languages, i, elt)
+		{
+			if (elt->prob < mean) {
+				g_ptr_array_remove_index_fast(chain->languages, i);
+#ifdef EXTRA_LANGDET_DEBUG
+				msg_debug_lang_det_cfg("remove %s from %s; prob: %.4f; mean: %.4f, std: %.4f",
+									   elt->elt->name, chain->utf, elt->prob, mean, std);
+#endif
+			}
+		}
+	}
+	else {
+		/* We have a unique ngramm, increase its weight */
+		PTR_ARRAY_FOREACH(chain->languages, i, elt)
+		{
+			elt->prob *= 4.0;
+#ifdef EXTRA_LANGDET_DEBUG
+			msg_debug_lang_det_cfg("increase weight of %s in %s; prob: %.4f",
+								   elt->elt->name, chain->utf, elt->prob);
+#endif
+		}
+	}
+}
+
+static void
+rspamd_language_detector_dtor(struct rspamd_lang_detector *d)
+{
+	if (d) {
+		for (guint i = 0; i < RSPAMD_LANGUAGE_MAX; i++) {
+			kh_destroy(rspamd_trigram_hash, d->trigrams[i]);
+			rspamd_multipattern_destroy(d->stop_words[i].mp);
+			g_array_free(d->stop_words[i].ranges, TRUE);
+		}
+
+		if (d->languages) {
+			kh_destroy(rspamd_languages_hash, d->languages);
+		}
+
+		kh_destroy(rspamd_stopwords_hash, d->stop_words_norm);
+		rspamd_lang_detection_fasttext_destroy(d->fasttext_detector);
+	}
+}
+
+struct rspamd_lang_detector *
+rspamd_language_detector_init(struct rspamd_config *cfg)
+{
+	const ucl_object_t *section, *elt, *languages_enable = NULL,
+									   *languages_disable = NULL;
+	const gchar *languages_path = default_languages_path;
+	glob_t gl;
+	size_t i, short_text_limit = default_short_text_limit, total = 0;
+	UErrorCode uc_err = U_ZERO_ERROR;
+	GString *languages_pattern;
+	struct rspamd_ngramm_chain *chain, schain;
+	gchar *fname;
+	struct rspamd_lang_detector *ret = NULL;
+	struct ucl_parser *parser;
+	ucl_object_t *stop_words;
+	bool prefer_fasttext = true;
+
+	section = ucl_object_lookup(cfg->cfg_ucl_obj, "lang_detection");
+
+	if (section != NULL) {
+		elt = ucl_object_lookup(section, "languages");
+
+		if (elt) {
+			languages_path = ucl_object_tostring(elt);
+		}
+
+		elt = ucl_object_lookup(section, "short_text_limit");
+
+		if (elt) {
+			short_text_limit = ucl_object_toint(elt);
+		}
+
+		languages_enable = ucl_object_lookup(section, "languages_enable");
+		languages_disable = ucl_object_lookup(section, "languages_disable");
+
+		elt = ucl_object_lookup(section, "prefer_fasttext");
+		if (elt) {
+			prefer_fasttext = ucl_object_toboolean(elt);
+		}
+	}
+
+	languages_pattern = g_string_sized_new(PATH_MAX);
+	rspamd_printf_gstring(languages_pattern, "%s/stop_words", languages_path);
+	parser = ucl_parser_new(UCL_PARSER_DEFAULT);
+
+	if (ucl_parser_add_file(parser, languages_pattern->str)) {
+		stop_words = ucl_parser_get_object(parser);
+	}
+	else {
+		msg_err_config("cannot read stop words from %s: %s",
+					   languages_pattern->str,
+					   ucl_parser_get_error(parser));
+		stop_words = NULL;
+	}
+
+	ucl_parser_free(parser);
+	languages_pattern->len = 0;
+
+	rspamd_printf_gstring(languages_pattern, "%s/*.json", languages_path);
+	memset(&gl, 0, sizeof(gl));
+
+	if (glob(languages_pattern->str, 0, NULL, &gl) != 0) {
+		msg_err_config("cannot read any files matching %v", languages_pattern);
+		goto end;
+	}
+
+	ret = rspamd_mempool_alloc0(cfg->cfg_pool, sizeof(*ret));
+	ret->languages = kh_init(rspamd_languages_hash);
+	kh_resize(rspamd_languages_hash, ret->languages, gl.gl_pathc);
+	ret->uchar_converter = rspamd_get_utf8_converter();
+	ret->short_text_limit = short_text_limit;
+	ret->stop_words_norm = kh_init(rspamd_stopwords_hash);
+	ret->prefer_fasttext = prefer_fasttext;
+
+	/* Map from ngramm in ucs32 to GPtrArray of rspamd_language_elt */
+	for (i = 0; i < RSPAMD_LANGUAGE_MAX; i++) {
+		ret->trigrams[i] = kh_init(rspamd_trigram_hash);
+#ifdef WITH_HYPERSCAN
+		ret->stop_words[i].mp = rspamd_multipattern_create(
+			RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8 |
+			RSPAMD_MULTIPATTERN_RE);
+#else
+		ret->stop_words[i].mp = rspamd_multipattern_create(
+			RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+#endif
+
+		ret->stop_words[i].ranges = g_array_new(FALSE, FALSE,
+												sizeof(struct rspamd_stop_word_range));
+	}
+
+	g_assert(uc_err == U_ZERO_ERROR);
+
+	for (i = 0; i < gl.gl_pathc; i++) {
+		fname = g_path_get_basename(gl.gl_pathv[i]);
+
+		if (!rspamd_ucl_array_find_str(fname, languages_disable) ||
+			(languages_enable == NULL ||
+			 rspamd_ucl_array_find_str(fname, languages_enable))) {
+			rspamd_language_detector_read_file(cfg, ret, gl.gl_pathv[i],
+											   stop_words);
+		}
+		else {
+			msg_info_config("skip language file %s: disabled", fname);
+		}
+
+		g_free(fname);
+	}
+
+	for (i = 0; i < RSPAMD_LANGUAGE_MAX; i++) {
+		GError *err = NULL;
+
+		kh_foreach_value(ret->trigrams[i], schain, {
+			chain = &schain;
+			rspamd_language_detector_process_chain(cfg, chain);
+		});
+
+		if (!rspamd_multipattern_compile(ret->stop_words[i].mp, &err)) {
+			msg_err_config("cannot compile stop words for %z language group: %e",
+						   i, err);
+			g_error_free(err);
+		}
+
+		total += kh_size(ret->trigrams[i]);
+	}
+
+	ret->fasttext_detector = rspamd_lang_detection_fasttext_init(cfg);
+	char *fasttext_status = rspamd_lang_detection_fasttext_show_info(ret->fasttext_detector);
+
+	msg_info_config("loaded %d languages, "
+					"%d trigrams; %s",
+					(gint) kh_size(ret->languages),
+					(gint) total, fasttext_status);
+	g_free(fasttext_status);
+
+	if (stop_words) {
+		ucl_object_unref(stop_words);
+	}
+
+	REF_INIT_RETAIN(ret, rspamd_language_detector_dtor);
+	rspamd_mempool_add_destructor(cfg->cfg_pool,
+								  (rspamd_mempool_destruct_t) rspamd_language_detector_unref,
+								  ret);
+
+end:
+	if (gl.gl_pathc > 0) {
+		globfree(&gl);
+	}
+
+	g_string_free(languages_pattern, TRUE);
+
+	return ret;
+}
+
+static void
+rspamd_language_detector_random_select(GArray *ucs_tokens, guint nwords,
+									   goffset *offsets_out,
+									   guint64 *seed)
+{
+	guint step_len, remainder, i, out_idx;
+	guint64 coin, sel;
+	rspamd_stat_token_t *tok;
+
+	g_assert(nwords != 0);
+	g_assert(offsets_out != NULL);
+	g_assert(ucs_tokens->len >= nwords);
+	/*
+	 * We split input array into `nwords` parts. For each part we randomly select
+	 * an element from this particular split. Here is an example:
+	 *
+	 * nwords=2, input_len=5
+	 *
+	 * w1 w2 w3   w4 w5
+	 * ^          ^
+	 * part1      part2
+	 *  vv         vv
+	 *  w2         w5
+	 *
+	 * So we have 2 output words from 5 input words selected randomly within
+	 * their splits. It is not uniform distribution but it seems to be better
+	 * to include words from different text parts
+	 */
+	step_len = ucs_tokens->len / nwords;
+	remainder = ucs_tokens->len % nwords;
+
+	out_idx = 0;
+	coin = rspamd_random_uint64_fast_seed(seed);
+	sel = coin % (step_len + remainder);
+	offsets_out[out_idx] = sel;
+
+	for (i = step_len + remainder; i < ucs_tokens->len;
+		 i += step_len, out_idx++) {
+		guint ntries = 0;
+		coin = rspamd_random_uint64_fast_seed(seed);
+		sel = (coin % step_len) + i;
+
+		for (;;) {
+			tok = &g_array_index(ucs_tokens, rspamd_stat_token_t, sel);
+			/* Filter bad tokens */
+
+			if (tok->unicode.len >= 2 &&
+				!(tok->flags & RSPAMD_STAT_TOKEN_FLAG_EXCEPTION) &&
+				u_isalpha(tok->unicode.begin[0]) &&
+				u_isalpha(tok->unicode.begin[tok->unicode.len - 1])) {
+				offsets_out[out_idx] = sel;
+				break;
+			}
+			else {
+				ntries++;
+				coin = rspamd_random_uint64_fast_seed(seed);
+
+				if (ntries < step_len) {
+					sel = (coin % step_len) + i;
+				}
+				else if (ntries < ucs_tokens->len) {
+					sel = coin % ucs_tokens->len;
+				}
+				else {
+					offsets_out[out_idx] = sel;
+					break;
+				}
+			}
+		}
+	}
+
+	/*
+	 * Fisher-Yates algorithm:
+	 * for i from 0 to n−2 do
+	 *   j ← random integer such that i ≤ j < n
+	 *   exchange a[i] and a[j]
+	 */
+#if 0
+	if (out_idx > 2) {
+		for (i = 0; i < out_idx - 2; i++) {
+			coin = rspamd_random_uint64_fast ();
+			sel = (coin % (out_idx - i)) + i;
+			/* swap */
+			tmp = offsets_out[i];
+			offsets_out[i] = offsets_out[sel];
+			offsets_out[sel] = tmp;
+		}
+	}
+#endif
+}
+
+static goffset
+rspamd_language_detector_next_ngramm(rspamd_stat_token_t *tok, UChar32 *window,
+									 guint wlen, goffset cur_off)
+{
+	guint i;
+
+	if (wlen > 1) {
+		/* Deal with spaces at the beginning and ending */
+
+		if (cur_off == 0) {
+			window[0] = (UChar32) ' ';
+
+			for (i = 0; i < wlen - 1; i++) {
+				window[i + 1] = tok->unicode.begin[i];
+			}
+		}
+		else if (cur_off + wlen == tok->unicode.len + 1) {
+			/* Add trailing space */
+			for (i = 0; i < wlen - 1; i++) {
+				window[i] = tok->unicode.begin[cur_off + i];
+			}
+			window[wlen - 1] = (UChar32) ' ';
+		}
+		else if (cur_off + wlen > tok->unicode.len + 1) {
+			/* No more fun */
+			return -1;
+		}
+		else {
+			/* Normal case */
+			for (i = 0; i < wlen; i++) {
+				window[i] = tok->unicode.begin[cur_off + i];
+			}
+		}
+	}
+	else {
+		if (tok->normalized.len <= cur_off) {
+			return -1;
+		}
+
+		window[0] = tok->unicode.begin[cur_off];
+	}
+
+	return cur_off + 1;
+}
+
+/*
+ * Do full guess for a specific ngramm, checking all languages defined
+ */
+static void
+rspamd_language_detector_process_ngramm_full(struct rspamd_task *task,
+											 struct rspamd_lang_detector *d,
+											 UChar32 *window,
+											 khash_t(rspamd_candidates_hash) * candidates,
+											 khash_t(rspamd_trigram_hash) * trigrams)
+{
+	guint i;
+	gint ret;
+	struct rspamd_ngramm_chain *chain = NULL;
+	struct rspamd_ngramm_elt *elt;
+	struct rspamd_lang_detector_res *cand;
+	khiter_t k;
+	gdouble prob;
+
+	k = kh_get(rspamd_trigram_hash, trigrams, window);
+	if (k != kh_end(trigrams)) {
+		chain = &kh_value(trigrams, k);
+	}
+
+	if (chain) {
+		PTR_ARRAY_FOREACH(chain->languages, i, elt)
+		{
+			prob = elt->prob;
+
+			if (prob < chain->mean) {
+				continue;
+			}
+
+			k = kh_get(rspamd_candidates_hash, candidates, elt->elt->name);
+			if (k != kh_end(candidates)) {
+				cand = kh_value(candidates, k);
+			}
+			else {
+				cand = NULL;
+			}
+
+#ifdef NGRAMMS_DEBUG
+			msg_err("gramm: %s, lang: %s, prob: %.3f", chain->utf,
+					elt->elt->name, log2(elt->prob));
+#endif
+			if (cand == NULL) {
+				cand = rspamd_mempool_alloc(task->task_pool, sizeof(*cand));
+				cand->elt = elt->elt;
+				cand->lang = elt->elt->name;
+				cand->prob = prob;
+
+				k = kh_put(rspamd_candidates_hash, candidates, elt->elt->name,
+						   &ret);
+				kh_value(candidates, k) = cand;
+			}
+			else {
+				/* Update guess */
+				cand->prob += prob;
+			}
+		}
+	}
+}
+
+static void
+rspamd_language_detector_detect_word(struct rspamd_task *task,
+									 struct rspamd_lang_detector *d,
+									 rspamd_stat_token_t *tok,
+									 khash_t(rspamd_candidates_hash) * candidates,
+									 khash_t(rspamd_trigram_hash) * trigrams)
+{
+	const guint wlen = 3;
+	UChar32 window[3];
+	goffset cur = 0;
+
+	/* Split words */
+	while ((cur = rspamd_language_detector_next_ngramm(tok, window, wlen, cur)) != -1) {
+		rspamd_language_detector_process_ngramm_full(task,
+													 d, window, candidates, trigrams);
+	}
+}
+
+static const gdouble cutoff_limit = -8.0;
+/*
+ * Converts frequencies to log probabilities, filter those candidates who
+ * has the lowest probabilities
+ */
+
+static inline void
+rspamd_language_detector_filter_step1(struct rspamd_task *task,
+									  struct rspamd_lang_detector_res *cand,
+									  gdouble *max_prob, guint *filtered)
+{
+	if (!isnan(cand->prob)) {
+		if (cand->prob == 0) {
+			cand->prob = NAN;
+			msg_debug_lang_det(
+				"exclude language %s",
+				cand->lang);
+			(*filtered)++;
+		}
+		else {
+			cand->prob = log2(cand->prob);
+			if (cand->prob < cutoff_limit) {
+				msg_debug_lang_det(
+					"exclude language %s: %.3f, cutoff limit: %.3f",
+					cand->lang, cand->prob, cutoff_limit);
+				cand->prob = NAN;
+				(*filtered)++;
+			}
+			else if (cand->prob > *max_prob) {
+				*max_prob = cand->prob;
+			}
+		}
+	}
+}
+
+static inline void
+rspamd_language_detector_filter_step2(struct rspamd_task *task,
+									  struct rspamd_lang_detector_res *cand,
+									  gdouble max_prob, guint *filtered)
+{
+	/*
+		 * Probabilities are logarithmic, so if prob1 - prob2 > 4, it means that
+		 * prob2 is 2^4 less than prob1
+		 */
+	if (!isnan(cand->prob) && max_prob - cand->prob > 1) {
+		msg_debug_lang_det("exclude language %s: %.3f (%.3f max)",
+						   cand->lang, cand->prob, max_prob);
+		cand->prob = NAN;
+		(*filtered)++;
+	}
+}
+
+static void
+rspamd_language_detector_filter_negligible(struct rspamd_task *task,
+										   khash_t(rspamd_candidates_hash) * candidates)
+{
+	struct rspamd_lang_detector_res *cand;
+	guint filtered = 0;
+	gdouble max_prob = -(G_MAXDOUBLE);
+
+	kh_foreach_value(candidates, cand,
+					 rspamd_language_detector_filter_step1(task, cand, &max_prob, &filtered));
+	kh_foreach_value(candidates, cand,
+					 rspamd_language_detector_filter_step2(task, cand, max_prob, &filtered));
+
+	msg_debug_lang_det("removed %d languages", filtered);
+}
+
+static void
+rspamd_language_detector_detect_type(struct rspamd_task *task,
+									 guint nwords,
+									 struct rspamd_lang_detector *d,
+									 GArray *words,
+									 enum rspamd_language_category cat,
+									 khash_t(rspamd_candidates_hash) * candidates,
+									 struct rspamd_mime_text_part *part)
+{
+	guint nparts = MIN(words->len, nwords);
+	goffset *selected_words;
+	rspamd_stat_token_t *tok;
+	guint i;
+	guint64 seed;
+
+	/* Seed PRNG with part digest to provide some sort of determinism */
+	memcpy(&seed, part->mime_part->digest, sizeof(seed));
+	selected_words = g_new0(goffset, nparts);
+	rspamd_language_detector_random_select(words, nparts, selected_words, &seed);
+	msg_debug_lang_det("randomly selected %d words", nparts);
+
+	for (i = 0; i < nparts; i++) {
+		tok = &g_array_index(words, rspamd_stat_token_t,
+							 selected_words[i]);
+
+		if (tok->unicode.len >= 3) {
+			rspamd_language_detector_detect_word(task, d, tok, candidates,
+												 d->trigrams[cat]);
+		}
+	}
+
+	/* Filter negligible candidates */
+	rspamd_language_detector_filter_negligible(task, candidates);
+	g_free(selected_words);
+}
+
+static gint
+rspamd_language_detector_cmp(gconstpointer a, gconstpointer b)
+{
+	const struct rspamd_lang_detector_res
+		*canda = *(const struct rspamd_lang_detector_res **) a,
+		*candb = *(const struct rspamd_lang_detector_res **) b;
+
+	if (canda->prob > candb->prob) {
+		return -1;
+	}
+	else if (candb->prob > canda->prob) {
+		return 1;
+	}
+
+	return 0;
+}
+
+enum rspamd_language_detected_type {
+	rs_detect_none = 0,
+	rs_detect_single,
+	rs_detect_multiple,
+};
+
+static enum rspamd_language_detected_type
+rspamd_language_detector_try_ngramm(struct rspamd_task *task,
+									guint nwords,
+									struct rspamd_lang_detector *d,
+									GArray *ucs_tokens,
+									enum rspamd_language_category cat,
+									khash_t(rspamd_candidates_hash) * candidates,
+									struct rspamd_mime_text_part *part)
+{
+	guint cand_len = 0;
+	struct rspamd_lang_detector_res *cand;
+
+	rspamd_language_detector_detect_type(task,
+										 nwords,
+										 d,
+										 ucs_tokens,
+										 cat,
+										 candidates,
+										 part);
+
+	kh_foreach_value(candidates, cand, {
+		if (!isnan(cand->prob)) {
+			cand_len++;
+		}
+	});
+
+	if (cand_len == 0) {
+		return rs_detect_none;
+	}
+	else if (cand_len == 1) {
+		return rs_detect_single;
+	}
+
+	return rs_detect_multiple;
+}
+
+enum rspamd_language_sort_flags {
+	RSPAMD_LANG_FLAG_DEFAULT = 0,
+	RSPAMD_LANG_FLAG_SHORT = 1 << 0,
+};
+
+struct rspamd_frequency_sort_cbdata {
+	struct rspamd_lang_detector *d;
+	enum rspamd_language_sort_flags flags;
+	gdouble std;
+	gdouble mean;
+};
+
+static const gdouble tier0_adjustment = 1.2;
+static const gdouble tier1_adjustment = 0.8;
+static const gdouble frequency_adjustment = 0.8;
+
+static gint
+rspamd_language_detector_cmp_heuristic(gconstpointer a, gconstpointer b,
+									   gpointer ud)
+{
+	struct rspamd_frequency_sort_cbdata *cbd = ud;
+	struct rspamd_lang_detector_res
+		*canda = *(struct rspamd_lang_detector_res **) a,
+		*candb = *(struct rspamd_lang_detector_res **) b;
+	gdouble adj;
+	gdouble proba_adjusted, probb_adjusted, freqa, freqb;
+
+	if (cbd->d->total_occurrences == 0) {
+		/* Not enough data, compare directly */
+		return rspamd_language_detector_cmp(a, b);
+	}
+
+	freqa = ((gdouble) canda->elt->occurrences) /
+			(gdouble) cbd->d->total_occurrences;
+	freqb = ((gdouble) candb->elt->occurrences) /
+			(gdouble) cbd->d->total_occurrences;
+
+	proba_adjusted = canda->prob;
+	probb_adjusted = candb->prob;
+
+	if (isnormal(freqa) && isnormal(freqb)) {
+		proba_adjusted += cbd->std * (frequency_adjustment * freqa);
+		probb_adjusted += cbd->std * (frequency_adjustment * freqb);
+	}
+
+	if (cbd->flags & RSPAMD_LANG_FLAG_SHORT) {
+		adj = tier1_adjustment * 2.0;
+	}
+	else {
+		adj = tier1_adjustment;
+	}
+	if (canda->elt->flags & RS_LANGUAGE_TIER1) {
+		proba_adjusted += cbd->std * adj;
+	}
+
+	if (candb->elt->flags & RS_LANGUAGE_TIER1) {
+		probb_adjusted += cbd->std * adj;
+	}
+
+	if (cbd->flags & RSPAMD_LANG_FLAG_SHORT) {
+		adj = tier0_adjustment * 16.0;
+	}
+	else {
+		adj = tier0_adjustment;
+	}
+
+	if (canda->elt->flags & RS_LANGUAGE_TIER0) {
+		proba_adjusted += cbd->std * adj;
+	}
+
+	if (candb->elt->flags & RS_LANGUAGE_TIER0) {
+		probb_adjusted += cbd->std * adj;
+	}
+
+	/* Hack: adjust probability directly */
+	canda->prob = proba_adjusted;
+	candb->prob = probb_adjusted;
+
+	if (proba_adjusted > probb_adjusted) {
+		return -1;
+	}
+	else if (probb_adjusted > proba_adjusted) {
+		return 1;
+	}
+
+	return 0;
+}
+
+static void
+rspamd_language_detector_unicode_scripts(struct rspamd_task *task,
+										 struct rspamd_mime_text_part *part,
+										 guint *pchinese,
+										 guint *pspecial)
+{
+	const gchar *p = part->utf_stripped_content->data, *end;
+	guint i = 0, cnt = 0;
+	end = p + part->utf_stripped_content->len;
+	gint32 uc, sc;
+	guint nlatin = 0, nchinese = 0, nspecial = 0;
+	const guint cutoff_limit = 32;
+
+	while (p + i < end) {
+		U8_NEXT(p, i, part->utf_stripped_content->len, uc);
+
+		if (((gint32) uc) < 0) {
+			break;
+		}
+
+		if (u_isalpha(uc)) {
+			sc = ublock_getCode(uc);
+			cnt++;
+
+			switch (sc) {
+			case UBLOCK_BASIC_LATIN:
+			case UBLOCK_LATIN_1_SUPPLEMENT:
+				part->unicode_scripts |= RSPAMD_UNICODE_LATIN;
+				nlatin++;
+				break;
+			case UBLOCK_HEBREW:
+				part->unicode_scripts |= RSPAMD_UNICODE_HEBREW;
+				nspecial++;
+				break;
+			case UBLOCK_GREEK:
+				part->unicode_scripts |= RSPAMD_UNICODE_GREEK;
+				nspecial++;
+				break;
+			case UBLOCK_CYRILLIC:
+				part->unicode_scripts |= RSPAMD_UNICODE_CYRILLIC;
+				nspecial++;
+				break;
+			case UBLOCK_CJK_UNIFIED_IDEOGRAPHS:
+			case UBLOCK_CJK_COMPATIBILITY:
+			case UBLOCK_CJK_RADICALS_SUPPLEMENT:
+			case UBLOCK_CJK_UNIFIED_IDEOGRAPHS_EXTENSION_A:
+			case UBLOCK_CJK_UNIFIED_IDEOGRAPHS_EXTENSION_B:
+				part->unicode_scripts |= RSPAMD_UNICODE_CJK;
+				nchinese++;
+				break;
+			case UBLOCK_HIRAGANA:
+			case UBLOCK_KATAKANA:
+				part->unicode_scripts |= RSPAMD_UNICODE_JP;
+				nspecial++;
+				break;
+			case UBLOCK_HANGUL_JAMO:
+			case UBLOCK_HANGUL_COMPATIBILITY_JAMO:
+				part->unicode_scripts |= RSPAMD_UNICODE_HANGUL;
+				nspecial++;
+				break;
+			case UBLOCK_ARABIC:
+				part->unicode_scripts |= RSPAMD_UNICODE_ARABIC;
+				nspecial++;
+				break;
+			case UBLOCK_DEVANAGARI:
+				part->unicode_scripts |= RSPAMD_UNICODE_DEVANAGARI;
+				nspecial++;
+				break;
+			case UBLOCK_ARMENIAN:
+				part->unicode_scripts |= RSPAMD_UNICODE_ARMENIAN;
+				nspecial++;
+				break;
+			case UBLOCK_GEORGIAN:
+				part->unicode_scripts |= RSPAMD_UNICODE_GEORGIAN;
+				nspecial++;
+				break;
+			case UBLOCK_GUJARATI:
+				part->unicode_scripts |= RSPAMD_UNICODE_GUJARATI;
+				nspecial++;
+				break;
+			case UBLOCK_TELUGU:
+				part->unicode_scripts |= RSPAMD_UNICODE_TELUGU;
+				nspecial++;
+				break;
+			case UBLOCK_TAMIL:
+				part->unicode_scripts |= RSPAMD_UNICODE_TAMIL;
+				nspecial++;
+				break;
+			case UBLOCK_THAI:
+				part->unicode_scripts |= RSPAMD_UNICODE_THAI;
+				nspecial++;
+				break;
+			case RSPAMD_UNICODE_MALAYALAM:
+				part->unicode_scripts |= RSPAMD_UNICODE_MALAYALAM;
+				nspecial++;
+				break;
+			case RSPAMD_UNICODE_SINHALA:
+				part->unicode_scripts |= RSPAMD_UNICODE_SINHALA;
+				nspecial++;
+				break;
+			}
+		}
+
+		if (nspecial > cutoff_limit && nspecial > nlatin) {
+			break;
+		}
+		else if (nchinese > cutoff_limit && nchinese > nlatin) {
+			if (nspecial > 0) {
+				/* Likely japanese */
+				break;
+			}
+		}
+	}
+
+	msg_debug_lang_det("stop after checking %d characters, "
+					   "%d latin, %d special, %d chinese",
+					   cnt, nlatin, nspecial, nchinese);
+
+	*pchinese = nchinese;
+	*pspecial = nspecial;
+}
+
+static inline void
+rspamd_language_detector_set_language(struct rspamd_task *task,
+									  struct rspamd_mime_text_part *part,
+									  const gchar *code,
+									  struct rspamd_language_elt *elt)
+{
+	struct rspamd_lang_detector_res *r;
+
+	r = rspamd_mempool_alloc0(task->task_pool, sizeof(*r));
+	r->prob = 1.0;
+	r->lang = code;
+	r->elt = elt;
+
+	if (part->languages == NULL) {
+		part->languages = g_ptr_array_sized_new(1);
+	}
+
+	g_ptr_array_add(part->languages, r);
+	part->language = code;
+}
+
+static gboolean
+rspamd_language_detector_try_uniscript(struct rspamd_task *task,
+									   struct rspamd_mime_text_part *part,
+									   guint nchinese,
+									   guint nspecial)
+{
+	guint i;
+
+	for (i = 0; i < G_N_ELEMENTS(unicode_langs); i++) {
+		if (unicode_langs[i].unicode_code & part->unicode_scripts) {
+
+			if (unicode_langs[i].unicode_code != RSPAMD_UNICODE_JP) {
+				msg_debug_lang_det("set language based on unicode script %s",
+								   unicode_langs[i].lang);
+				rspamd_language_detector_set_language(task, part,
+													  unicode_langs[i].lang, NULL);
+
+				return TRUE;
+			}
+			else {
+				/* Japanese <-> Chinese guess */
+
+				/*
+				 * Typically there might be around 0-70% of kanji glyphs
+				 * and the rest are Haragana/Katakana
+				 *
+				 * If we discover that Kanji is more than 80% then we consider
+				 * it Chinese
+				 */
+				if (nchinese <= 5 || nchinese < nspecial * 5) {
+					msg_debug_lang_det("set language based on unicode script %s",
+									   unicode_langs[i].lang);
+					rspamd_language_detector_set_language(task, part,
+														  unicode_langs[i].lang, NULL);
+
+					return TRUE;
+				}
+			}
+		}
+	}
+
+	if (part->unicode_scripts & RSPAMD_UNICODE_CJK) {
+		msg_debug_lang_det("guess chinese based on CJK characters: %d chinese, %d special",
+						   nchinese, nspecial);
+		rspamd_language_detector_set_language(task, part,
+											  "zh-CN", NULL);
+
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static guint
+rspamd_langelt_hash_func(gconstpointer key)
+{
+	const struct rspamd_language_elt *elt = (const struct rspamd_language_elt *) key;
+	return rspamd_cryptobox_fast_hash(elt->name, strlen(elt->name),
+									  rspamd_hash_seed());
+}
+
+static gboolean
+rspamd_langelt_equal_func(gconstpointer v, gconstpointer v2)
+{
+	const struct rspamd_language_elt *elt1 = (const struct rspamd_language_elt *) v,
+									 *elt2 = (const struct rspamd_language_elt *) v2;
+	return strcmp(elt1->name, elt2->name) == 0;
+}
+
+/* This hash set stores a word index in the language to avoid duplicate stop words */
+KHASH_INIT(rspamd_sw_res_set, int, char, 0, kh_int_hash_func, kh_int_hash_equal);
+
+KHASH_INIT(rspamd_sw_hash, struct rspamd_language_elt *, khash_t(rspamd_sw_res_set) *, 1,
+		   rspamd_langelt_hash_func, rspamd_langelt_equal_func);
+
+struct rspamd_sw_cbdata {
+	struct rspamd_task *task;
+	khash_t(rspamd_sw_hash) * res;
+	GArray *ranges;
+};
+
+static gint
+rspamd_ranges_cmp(const void *k, const void *memb)
+{
+	gint pos = GPOINTER_TO_INT(k);
+	const struct rspamd_stop_word_range *r = (struct rspamd_stop_word_range *) memb;
+
+	if (pos >= r->start && pos < r->stop) {
+		return 0;
+	}
+	else if (pos < r->start) {
+		return -1;
+	}
+
+	return 1;
+}
+
+static gint
+rspamd_language_detector_sw_cb(struct rspamd_multipattern *mp,
+							   guint strnum,
+							   gint match_start,
+							   gint match_pos,
+							   const gchar *text,
+							   gsize len,
+							   void *context)
+{
+	/* Check if boundary */
+	const gchar *prev = text, *next = text + len;
+	struct rspamd_stop_word_range *r;
+	struct rspamd_sw_cbdata *cbdata = (struct rspamd_sw_cbdata *) context;
+	khiter_t k;
+	static const gsize max_stop_words = 80;
+	struct rspamd_task *task;
+
+	if (match_start > 0) {
+		prev = text + match_start - 1;
+
+		if (!(g_ascii_isspace(*prev) || g_ascii_ispunct(*prev))) {
+			return 0;
+		}
+	}
+
+	if (match_pos < len) {
+		next = text + match_pos;
+
+		if (!(g_ascii_isspace(*next) || g_ascii_ispunct(*next))) {
+			return 0;
+		}
+	}
+
+	/* We have a word on the boundary, check range */
+	task = cbdata->task;
+	r = bsearch(GINT_TO_POINTER(strnum), cbdata->ranges->data,
+				cbdata->ranges->len, sizeof(*r), rspamd_ranges_cmp);
+
+	g_assert(r != NULL);
+
+	k = kh_get(rspamd_sw_hash, cbdata->res, r->elt);
+	gint nwords = 1;
+
+	if (k != kh_end(cbdata->res)) {
+		khiter_t set_k;
+		int tt;
+
+		set_k = kh_get(rspamd_sw_res_set, kh_value(cbdata->res, k), strnum);
+		nwords = kh_size(kh_value(cbdata->res, k));
+
+		if (set_k == kh_end(kh_value(cbdata->res, k))) {
+			/* New word */
+			set_k = kh_put(rspamd_sw_res_set, kh_value(cbdata->res, k), strnum, &tt);
+			msg_debug_lang_det("found new word %*s from %s language (%d stop words found so far)",
+							   (int) (next - prev - 1), prev + 1, r->elt->name, nwords);
+		}
+
+		if (nwords > max_stop_words) {
+			return 1;
+		}
+	}
+	else {
+		gint tt;
+
+		k = kh_put(rspamd_sw_hash, cbdata->res, r->elt, &tt);
+		kh_value(cbdata->res, k) = kh_init(rspamd_sw_res_set);
+		kh_put(rspamd_sw_res_set, kh_value(cbdata->res, k), strnum, &tt);
+
+		msg_debug_lang_det("found new word %*s from %s language (%d stop words found so far)",
+						   (int) (next - prev - 1), prev + 1, r->elt->name, nwords);
+	}
+
+	return 0;
+}
+
+static gboolean
+rspamd_language_detector_try_stop_words(struct rspamd_task *task,
+										struct rspamd_lang_detector *d,
+										struct rspamd_mime_text_part *part,
+										enum rspamd_language_category cat)
+{
+	struct rspamd_stop_word_elt *elt;
+	struct rspamd_sw_cbdata cbdata;
+	gboolean ret = FALSE;
+	static const int stop_words_threshold = 4, /* minimum stop words count */
+		strong_confidence_threshold = 10 /* we are sure that this is enough */;
+
+	elt = &d->stop_words[cat];
+	cbdata.res = kh_init(rspamd_sw_hash);
+	cbdata.ranges = elt->ranges;
+	cbdata.task = task;
+
+	rspamd_multipattern_lookup(elt->mp, part->utf_stripped_content->data,
+							   part->utf_stripped_content->len, rspamd_language_detector_sw_cb,
+							   &cbdata, NULL);
+
+	if (kh_size(cbdata.res) > 0) {
+		khash_t(rspamd_sw_res_set) * cur_res;
+		double max_rate = G_MINDOUBLE;
+		struct rspamd_language_elt *cur_lang, *sel = NULL;
+		gboolean ignore_ascii = FALSE, ignore_latin = FALSE;
+
+	again:
+		kh_foreach(cbdata.res, cur_lang, cur_res, {
+			int cur_matches = kh_size(cur_res);
+
+			if (!ignore_ascii && (cur_lang->flags & RS_LANGUAGE_DIACRITICS)) {
+				/* Restart matches */
+				ignore_ascii = TRUE;
+				sel = NULL;
+				max_rate = G_MINDOUBLE;
+				msg_debug_lang_det("ignore ascii after finding %d stop words from %s",
+								   cur_matches, cur_lang->name);
+				goto again;
+			}
+
+			if (!ignore_latin && cur_lang->category != RSPAMD_LANGUAGE_LATIN) {
+				/* Restart matches */
+				ignore_latin = TRUE;
+				sel = NULL;
+				max_rate = G_MINDOUBLE;
+				msg_debug_lang_det("ignore latin after finding stop %d words from %s",
+								   cur_matches, cur_lang->name);
+				goto again;
+			}
+
+			if (cur_matches < stop_words_threshold) {
+				continue;
+			}
+
+			if (cur_matches < strong_confidence_threshold) {
+				/* Ignore mixed languages when not enough confidence */
+				if (ignore_ascii && (cur_lang->flags & RS_LANGUAGE_ASCII)) {
+					continue;
+				}
+
+				if (ignore_latin && cur_lang->category == RSPAMD_LANGUAGE_LATIN) {
+					continue;
+				}
+			}
+
+			double rate = (double) cur_matches / (double) cur_lang->stop_words;
+
+			if (rate > max_rate) {
+				max_rate = rate;
+				sel = cur_lang;
+			}
+
+			msg_debug_lang_det("found %d stop words from %s: %3f rate",
+							   cur_matches, cur_lang->name, rate);
+		});
+
+		/* Cleanup */
+		kh_foreach(cbdata.res, cur_lang, cur_res, {
+			kh_destroy(rspamd_sw_res_set, cur_res);
+		});
+
+		if (max_rate > 0 && sel) {
+			msg_debug_lang_det("set language based on stop words script %s, %.3f found",
+							   sel->name, max_rate);
+			rspamd_language_detector_set_language(task, part,
+												  sel->name, sel);
+
+			ret = TRUE;
+		}
+	}
+	else {
+		msg_debug_lang_det("found no stop words in a text");
+	}
+
+	kh_destroy(rspamd_sw_hash, cbdata.res);
+
+	return ret;
+}
+
+gboolean
+rspamd_language_detector_detect(struct rspamd_task *task,
+								struct rspamd_lang_detector *d,
+								struct rspamd_mime_text_part *part)
+{
+	khash_t(rspamd_candidates_hash) * candidates;
+	GPtrArray *result;
+	gdouble mean, std, start_ticks, end_ticks;
+	guint cand_len;
+	enum rspamd_language_category cat;
+	struct rspamd_lang_detector_res *cand;
+	enum rspamd_language_detected_type r;
+	struct rspamd_frequency_sort_cbdata cbd;
+	/* Check if we have sorted candidates based on frequency */
+	gboolean frequency_heuristic_applied = FALSE, ret = FALSE;
+
+	if (!part->utf_stripped_content) {
+		return FALSE;
+	}
+
+	start_ticks = rspamd_get_ticks(TRUE);
+
+	guint nchinese = 0, nspecial = 0;
+	rspamd_language_detector_unicode_scripts(task, part, &nchinese, &nspecial);
+
+	/* Disable internal language detection heuristics if we have fasttext */
+	if (!rspamd_lang_detection_fasttext_is_enabled(d->fasttext_detector) || !d->prefer_fasttext) {
+		/* Apply unicode scripts heuristic */
+		if (rspamd_language_detector_try_uniscript(task, part, nchinese, nspecial)) {
+			ret = TRUE;
+		}
+
+		cat = rspamd_language_detector_get_category(part->unicode_scripts);
+
+		if (!ret && rspamd_language_detector_try_stop_words(task, d, part, cat)) {
+			ret = TRUE;
+		}
+	}
+
+	if (!ret) {
+		unsigned ndetected = 0;
+		if (rspamd_lang_detection_fasttext_is_enabled(d->fasttext_detector)) {
+			rspamd_fasttext_predict_result_t fasttext_predict_result =
+				rspamd_lang_detection_fasttext_detect(d->fasttext_detector, task,
+													  part->utf_words, 4);
+
+			ndetected = rspamd_lang_detection_fasttext_get_nlangs(fasttext_predict_result);
+
+			if (ndetected > 0) {
+				candidates = kh_init(rspamd_candidates_hash);
+				kh_resize(rspamd_candidates_hash, candidates, ndetected);
+
+				/* Now fill all results where probability is above threshold */
+				float max_prob = rspamd_lang_detection_fasttext_get_prob(fasttext_predict_result, 0);
+
+				for (unsigned int i = 0; i < ndetected; i++) {
+					float prob = rspamd_lang_detection_fasttext_get_prob(fasttext_predict_result, i);
+					if (prob > max_prob * 0.75) {
+						char *lang = rspamd_mempool_strdup(task->task_pool,
+														   rspamd_lang_detection_fasttext_get_lang(fasttext_predict_result, i));
+						int tmp;
+						khiter_t k = kh_put(rspamd_candidates_hash, candidates, lang, &tmp);
+
+						kh_value(candidates, k) = rspamd_mempool_alloc0(task->task_pool, sizeof(*cand));
+						cand = kh_value(candidates, k);
+						cand->lang = lang;
+						cand->prob = rspamd_lang_detection_fasttext_get_prob(fasttext_predict_result, i);
+
+						/* Find the corresponding language elt */
+						k = kh_get(rspamd_languages_hash, d->languages, lang);
+						if (k != kh_end(d->languages)) {
+							cand->elt = kh_value(d->languages, k);
+						}
+					}
+				}
+
+				if (kh_size(candidates) == 1) {
+					r = rs_detect_single;
+				}
+				else if (kh_size(candidates) > 1) {
+					r = rs_detect_multiple;
+				}
+				else {
+					r = rs_detect_none;
+				}
+			}
+
+			rspamd_fasttext_predict_result_destroy(fasttext_predict_result);
+		}
+		if (ndetected == 0) {
+			if (part->utf_words->len < default_short_text_limit) {
+				r = rs_detect_none;
+				msg_debug_lang_det("text is too short for trigrams detection: "
+								   "%d words; at least %d words required",
+								   (int) part->utf_words->len,
+								   (int) default_short_text_limit);
+				switch (cat) {
+				case RSPAMD_LANGUAGE_CYRILLIC:
+					rspamd_language_detector_set_language(task, part, "ru", NULL);
+					break;
+				case RSPAMD_LANGUAGE_DEVANAGARI:
+					rspamd_language_detector_set_language(task, part, "hi", NULL);
+					break;
+				case RSPAMD_LANGUAGE_ARAB:
+					rspamd_language_detector_set_language(task, part, "ar", NULL);
+					break;
+				default:
+				case RSPAMD_LANGUAGE_LATIN:
+					rspamd_language_detector_set_language(task, part, "en", NULL);
+					break;
+				}
+				msg_debug_lang_det("set %s language based on symbols category",
+								   part->language);
+
+				candidates = kh_init(rspamd_candidates_hash);
+			}
+			else {
+				candidates = kh_init(rspamd_candidates_hash);
+				kh_resize(rspamd_candidates_hash, candidates, 32);
+
+				r = rspamd_language_detector_try_ngramm(task,
+														default_words,
+														d,
+														part->utf_words,
+														cat,
+														candidates,
+														part);
+
+				if (r == rs_detect_none) {
+					msg_debug_lang_det("no trigrams found, fallback to english");
+					rspamd_language_detector_set_language(task, part, "en", NULL);
+				}
+				else if (r == rs_detect_multiple) {
+					/* Check our guess */
+
+					mean = 0.0;
+					std = 0.0;
+					cand_len = 0;
+
+					/* Check distribution */
+					kh_foreach_value(candidates, cand, {
+						if (!isnan(cand->prob)) {
+							mean += cand->prob;
+							cand_len++;
+						}
+					});
+
+					if (cand_len > 0) {
+						mean /= cand_len;
+
+						kh_foreach_value(candidates, cand, {
+							gdouble err;
+							if (!isnan(cand->prob)) {
+								err = cand->prob - mean;
+								std += fabs(err);
+							}
+						});
+
+						std /= cand_len;
+					}
+
+					msg_debug_lang_det("trigrams checked, %d candidates, %.3f mean, %.4f stddev",
+									   cand_len, mean, std);
+
+					if (cand_len > 0 && std / fabs(mean) < 0.25) {
+						msg_debug_lang_det("apply frequency heuristic sorting");
+						frequency_heuristic_applied = TRUE;
+						cbd.d = d;
+						cbd.mean = mean;
+						cbd.std = std;
+						cbd.flags = RSPAMD_LANG_FLAG_DEFAULT;
+
+						if (part->nwords < default_words / 2) {
+							cbd.flags |= RSPAMD_LANG_FLAG_SHORT;
+						}
+					}
+				}
+			}
+		}
+
+		/* Now, convert hash to array and sort it */
+		if (r != rs_detect_none && kh_size(candidates) > 0) {
+			result = g_ptr_array_sized_new(kh_size(candidates));
+
+			kh_foreach_value(candidates, cand, {
+				if (!isnan(cand->prob)) {
+					msg_debug_lang_det("pre-sorting probability %s -> %.2f", cand->lang,
+									   cand->prob);
+					g_ptr_array_add(result, cand);
+				}
+			});
+
+			if (frequency_heuristic_applied) {
+				g_ptr_array_sort_with_data(result,
+										   rspamd_language_detector_cmp_heuristic,
+										   (gpointer) &cbd);
+			}
+			else {
+				g_ptr_array_sort(result, rspamd_language_detector_cmp);
+			}
+
+			int i;
+			PTR_ARRAY_FOREACH(result, i, cand)
+			{
+				msg_debug_lang_det("final probability %s -> %.2f", cand->lang,
+								   cand->prob);
+			}
+
+			if (part->languages != NULL) {
+				g_ptr_array_unref(part->languages);
+			}
+
+			part->languages = result;
+			part->language = ((struct rspamd_lang_detector_res *) g_ptr_array_index(result, 0))->lang;
+			ret = TRUE;
+		}
+		else if (part->languages == NULL) {
+			rspamd_language_detector_set_language(task, part, "en", NULL);
+		}
+
+		kh_destroy(rspamd_candidates_hash, candidates);
+	}
+
+	/* Update internal stat */
+	if (part->languages != NULL && part->languages->len > 0 && !frequency_heuristic_applied) {
+		cand = g_ptr_array_index(part->languages, 0);
+		if (cand->elt) {
+			cand->elt->occurrences++;
+			d->total_occurrences++;
+
+			msg_debug_lang_det("updated stat for %s: %d occurrences, %z total detected",
+							   cand->elt->name, cand->elt->occurrences,
+							   d->total_occurrences);
+		}
+	}
+
+	end_ticks = rspamd_get_ticks(TRUE);
+	msg_debug_lang_det("detected languages in %.0f ticks",
+					   (end_ticks - start_ticks));
+
+	return ret;
+}
+
+
+struct rspamd_lang_detector *
+rspamd_language_detector_ref(struct rspamd_lang_detector *d)
+{
+	REF_RETAIN(d);
+
+	return d;
+}
+
+void rspamd_language_detector_unref(struct rspamd_lang_detector *d)
+{
+	REF_RELEASE(d);
+}
+
+gboolean
+rspamd_language_detector_is_stop_word(struct rspamd_lang_detector *d,
+									  const gchar *word, gsize wlen)
+{
+	khiter_t k;
+	rspamd_ftok_t search;
+
+	search.begin = word;
+	search.len = wlen;
+
+	k = kh_get(rspamd_stopwords_hash, d->stop_words_norm, &search);
+
+	if (k != kh_end(d->stop_words_norm)) {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+gint rspamd_language_detector_elt_flags(const struct rspamd_language_elt *elt)
+{
+	if (elt) {
+		return elt->flags;
+	}
+
+	return 0;
+}
\ No newline at end of file
diff --git a/src/libmime/lang_detection.h b/src/libmime/lang_detection.h
new file mode 100644
index 0000000..5423c13
--- /dev/null
+++ b/src/libmime/lang_detection.h
@@ -0,0 +1,110 @@
+/*-
+ * Copyright 2017 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef RSPAMD_LANG_DETECTION_H
+#define RSPAMD_LANG_DETECTION_H
+
+#include "config.h"
+#include "libserver/cfg_file.h"
+#include "libstat/stat_api.h"
+#include "libmime/message.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_lang_detector;
+struct rspamd_language_elt;
+struct rspamd_task;
+
+enum rspamd_unicode_scripts {
+	RSPAMD_UNICODE_LATIN = (1 << 0),
+	RSPAMD_UNICODE_GREEK = (1 << 1),
+	RSPAMD_UNICODE_CYRILLIC = (1 << 2),
+	RSPAMD_UNICODE_HEBREW = (1 << 3),
+	RSPAMD_UNICODE_CJK = (1 << 4),
+	RSPAMD_UNICODE_JP = (1 << 5),
+	RSPAMD_UNICODE_ARABIC = (1 << 6),
+	RSPAMD_UNICODE_DEVANAGARI = (1 << 7),
+	RSPAMD_UNICODE_THAI = (1 << 8),
+	RSPAMD_UNICODE_ARMENIAN = (1 << 9),
+	RSPAMD_UNICODE_GEORGIAN = (1 << 10),
+	RSPAMD_UNICODE_GUJARATI = (1 << 11),
+	RSPAMD_UNICODE_TAMIL = (1 << 12),
+	RSPAMD_UNICODE_TELUGU = (1 << 13),
+	RSPAMD_UNICODE_MALAYALAM = (1 << 14),
+	RSPAMD_UNICODE_SINHALA = (1 << 15),
+	RSPAMD_UNICODE_HANGUL = (1 << 16),
+};
+
+enum rspamd_language_elt_flags {
+	RS_LANGUAGE_DEFAULT = 0,
+	RS_LANGUAGE_LATIN = (1 << 0),
+	RS_LANGUAGE_TIER1 = (1 << 3),
+	RS_LANGUAGE_TIER0 = (1 << 4),
+	RS_LANGUAGE_DIACRITICS = (1 << 5),
+	RS_LANGUAGE_ASCII = (1 << 6),
+};
+
+struct rspamd_lang_detector_res {
+	gdouble prob;
+	const gchar *lang;
+	struct rspamd_language_elt *elt;
+};
+
+/**
+ * Create new language detector object using configuration object
+ * @param cfg
+ * @return
+ */
+struct rspamd_lang_detector *rspamd_language_detector_init(struct rspamd_config *cfg);
+
+struct rspamd_lang_detector *rspamd_language_detector_ref(struct rspamd_lang_detector *d);
+
+void rspamd_language_detector_unref(struct rspamd_lang_detector *d);
+
+/**
+ * Try to detect language of words
+ * @param d
+ * @param ucs_tokens
+ * @param words_len
+ * @return array of struct rspamd_lang_detector_res sorted by freq descending
+ */
+gboolean rspamd_language_detector_detect(struct rspamd_task *task,
+										 struct rspamd_lang_detector *d,
+										 struct rspamd_mime_text_part *part);
+
+/**
+ * Returns TRUE if the specified word is known to be a stop word
+ * @param d
+ * @param word
+ * @param wlen
+ * @return
+ */
+gboolean rspamd_language_detector_is_stop_word(struct rspamd_lang_detector *d,
+											   const gchar *word, gsize wlen);
+
+/**
+ * Return language flags for a specific language elt
+ * @param elt
+ * @return
+ */
+gint rspamd_language_detector_elt_flags(const struct rspamd_language_elt *elt);
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libmime/lang_detection_fasttext.cxx b/src/libmime/lang_detection_fasttext.cxx
new file mode 100644
index 0000000..c973ed7
--- /dev/null
+++ b/src/libmime/lang_detection_fasttext.cxx
@@ -0,0 +1,269 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "lang_detection_fasttext.h"
+
+#ifdef WITH_FASTTEXT
+#include "fasttext/fasttext.h"
+#include "libserver/cfg_file.h"
+#include "libserver/logger.h"
+#include "fmt/core.h"
+#include "stat_api.h"
+#include <exception>
+#include <string_view>
+#include <vector>
+#endif
+
+#ifdef WITH_FASTTEXT
+
+EXTERN_LOG_MODULE_DEF(langdet);
+#define msg_debug_lang_det(...) rspamd_conditional_debug_fast(nullptr, nullptr,                                           \
+															  rspamd_langdet_log_id, "langdet", task->task_pool->tag.uid, \
+															  __FUNCTION__,                                               \
+															  __VA_ARGS__)
+
+namespace rspamd::langdet {
+class fasttext_langdet {
+private:
+	fasttext::FastText ft;
+	std::string model_fname;
+	bool loaded = false;
+
+public:
+	explicit fasttext_langdet(struct rspamd_config *cfg)
+	{
+		const auto *ucl_obj = cfg->cfg_ucl_obj;
+		const auto *opts_section = ucl_object_find_key(ucl_obj, "lang_detection");
+
+		if (opts_section) {
+			const auto *model = ucl_object_find_key(opts_section, "fasttext_model");
+
+			if (model) {
+				try {
+					ft.loadModel(ucl_object_tostring(model));
+					loaded = true;
+					model_fname = std::string{ucl_object_tostring(model)};
+				} catch (std::exception &e) {
+					auto err_message = fmt::format("cannot load fasttext model: {}", e.what());
+					msg_err_config("%s", err_message.c_str());
+					loaded = false;
+				}
+			}
+		}
+	}
+
+	/* Disallow multiple initialisation */
+	fasttext_langdet() = delete;
+	fasttext_langdet(const fasttext_langdet &) = delete;
+	fasttext_langdet(fasttext_langdet &&) = delete;
+
+	~fasttext_langdet() = default;
+
+	auto is_enabled() const -> bool
+	{
+		return loaded;
+	}
+	auto word2vec(const char *in, std::size_t len, std::vector<std::int32_t> &word_ngramms) const
+	{
+		if (!loaded) {
+			return;
+		}
+
+		std::string tok{in, len};
+		const auto &dic = ft.getDictionary();
+		auto h = dic->hash(tok);
+		auto wid = dic->getId(tok, h);
+		auto type = wid < 0 ? dic->getType(tok) : dic->getType(wid);
+
+		if (type == fasttext::entry_type::word) {
+			if (wid < 0) {
+				auto pipelined_word = fmt::format("{}{}{}", fasttext::Dictionary::BOW, tok, fasttext::Dictionary::EOW);
+				dic->computeSubwords(pipelined_word, word_ngramms);
+			}
+			else {
+				if (ft.getArgs().maxn <= 0) {
+					word_ngramms.push_back(wid);
+				}
+				else {
+					const auto ngrams = dic->getSubwords(wid);
+					word_ngramms.insert(word_ngramms.end(), ngrams.cbegin(), ngrams.cend());
+				}
+			}
+		}
+	}
+	auto detect_language(std::vector<std::int32_t> &words, int k)
+		-> std::vector<std::pair<fasttext::real, std::string>> *
+	{
+		if (!loaded) {
+			return nullptr;
+		}
+
+		auto predictions = new std::vector<std::pair<fasttext::real, std::string>>;
+		predictions->reserve(k);
+		fasttext::Predictions line_predictions;
+		line_predictions.reserve(k);
+		ft.predict(k, words, line_predictions, 0.0f);
+		const auto *dict = ft.getDictionary().get();
+
+		for (const auto &pred: line_predictions) {
+			predictions->push_back(std::make_pair(std::exp(pred.first), dict->getLabel(pred.second)));
+		}
+		return predictions;
+	}
+
+	auto model_info(void) const -> const std::string
+	{
+		if (!loaded) {
+			static const auto not_loaded = std::string{"fasttext model is not loaded"};
+			return not_loaded;
+		}
+		else {
+			return fmt::format("fasttext model {}: {} languages, {} tokens", model_fname,
+							   ft.getDictionary()->nlabels(), ft.getDictionary()->ntokens());
+		}
+	}
+};
+}// namespace rspamd::langdet
+#endif
+
+/* C API part */
+G_BEGIN_DECLS
+
+#define FASTTEXT_MODEL_TO_C_API(p) reinterpret_cast<rspamd::langdet::fasttext_langdet *>(p)
+#define FASTTEXT_RESULT_TO_C_API(res) reinterpret_cast<std::vector<std::pair<fasttext::real, std::string>> *>(res)
+
+void *rspamd_lang_detection_fasttext_init(struct rspamd_config *cfg)
+{
+#ifndef WITH_FASTTEXT
+	return nullptr;
+#else
+	return (void *) new rspamd::langdet::fasttext_langdet(cfg);
+#endif
+}
+
+char *rspamd_lang_detection_fasttext_show_info(void *ud)
+{
+#ifndef WITH_FASTTEXT
+	return g_strdup("fasttext is not compiled in");
+#else
+	auto model_info = FASTTEXT_MODEL_TO_C_API(ud)->model_info();
+
+	return g_strdup(model_info.c_str());
+#endif
+}
+
+bool rspamd_lang_detection_fasttext_is_enabled(void *ud)
+{
+#ifdef WITH_FASTTEXT
+	auto *real_model = FASTTEXT_MODEL_TO_C_API(ud);
+
+	if (real_model) {
+		return real_model->is_enabled();
+	}
+#endif
+
+	return false;
+}
+
+rspamd_fasttext_predict_result_t rspamd_lang_detection_fasttext_detect(void *ud,
+																	   struct rspamd_task *task,
+																	   GArray *utf_words,
+																	   int k)
+{
+#ifndef WITH_FASTTEXT
+	return nullptr;
+#else
+	/* Avoid too long inputs */
+	static const guint max_fasttext_input_len = 1024 * 1024;
+	auto *real_model = FASTTEXT_MODEL_TO_C_API(ud);
+	std::vector<std::int32_t> words_vec;
+	words_vec.reserve(utf_words->len);
+
+	for (auto i = 0; i < std::min(utf_words->len, max_fasttext_input_len); i++) {
+		const auto *w = &g_array_index(utf_words, rspamd_stat_token_t, i);
+		if (w->original.len > 0) {
+			real_model->word2vec(w->original.begin, w->original.len, words_vec);
+		}
+	}
+
+	msg_debug_lang_det("fasttext: got %z word tokens from %ud words", words_vec.size(), utf_words->len);
+
+	auto *res = real_model->detect_language(words_vec, k);
+
+	return (rspamd_fasttext_predict_result_t) res;
+#endif
+}
+
+void rspamd_lang_detection_fasttext_destroy(void *ud)
+{
+#ifdef WITH_FASTTEXT
+	delete FASTTEXT_MODEL_TO_C_API(ud);
+#endif
+}
+
+
+guint rspamd_lang_detection_fasttext_get_nlangs(rspamd_fasttext_predict_result_t res)
+{
+#ifdef WITH_FASTTEXT
+	auto *real_res = FASTTEXT_RESULT_TO_C_API(res);
+
+	if (real_res) {
+		return real_res->size();
+	}
+#endif
+	return 0;
+}
+
+const char *
+rspamd_lang_detection_fasttext_get_lang(rspamd_fasttext_predict_result_t res, unsigned int idx)
+{
+#ifdef WITH_FASTTEXT
+	auto *real_res = FASTTEXT_RESULT_TO_C_API(res);
+
+	if (real_res && real_res->size() > idx) {
+		/* Fasttext returns result in form __label__<lang>, so we need to remove __label__ prefix */
+		auto lang = std::string_view{real_res->at(idx).second};
+		if (lang.size() > sizeof("__label__") && lang.substr(0, sizeof("__label__") - 1) == "__label__") {
+			lang.remove_prefix(sizeof("__label__") - 1);
+		}
+		return lang.data();
+	}
+#endif
+	return nullptr;
+}
+
+float rspamd_lang_detection_fasttext_get_prob(rspamd_fasttext_predict_result_t res, unsigned int idx)
+{
+#ifdef WITH_FASTTEXT
+	auto *real_res = FASTTEXT_RESULT_TO_C_API(res);
+
+	if (real_res && real_res->size() > idx) {
+		return real_res->at(idx).first;
+	}
+#endif
+	return 0.0f;
+}
+
+void rspamd_fasttext_predict_result_destroy(rspamd_fasttext_predict_result_t res)
+{
+#ifdef WITH_FASTTEXT
+	auto *real_res = FASTTEXT_RESULT_TO_C_API(res);
+
+	delete real_res;
+#endif
+}
+
+G_END_DECLS
\ No newline at end of file
diff --git a/src/libmime/lang_detection_fasttext.h b/src/libmime/lang_detection_fasttext.h
new file mode 100644
index 0000000..c8710d3
--- /dev/null
+++ b/src/libmime/lang_detection_fasttext.h
@@ -0,0 +1,91 @@
+/*-
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_LANG_DETECTION_FASTTEXT_H
+#define RSPAMD_LANG_DETECTION_FASTTEXT_H
+
+#include "config.h"
+
+G_BEGIN_DECLS
+struct rspamd_config;
+struct rspamd_task; /* for logging */
+/**
+ * Initialize fasttext language detector
+ * @param cfg
+ * @return opaque pointer
+ */
+void *rspamd_lang_detection_fasttext_init(struct rspamd_config *cfg);
+
+/**
+ * Check if fasttext language detector is enabled
+ * @param ud
+ * @return
+ */
+bool rspamd_lang_detection_fasttext_is_enabled(void *ud);
+
+/**
+ * Show info about fasttext language detector
+ * @param ud
+ * @return
+ */
+char *rspamd_lang_detection_fasttext_show_info(void *ud);
+
+
+typedef void *rspamd_fasttext_predict_result_t;
+/**
+ * Detect language using fasttext
+ * @param ud opaque pointer
+ * @param in input text
+ * @param len length of input text
+ * @param k number of results to return
+ * @return TRUE if language is detected
+ */
+rspamd_fasttext_predict_result_t rspamd_lang_detection_fasttext_detect(void *ud,
+																	   struct rspamd_task *task, GArray *utf_words, int k);
+
+/**
+ * Get number of languages detected
+ * @param ud
+ * @return
+ */
+guint rspamd_lang_detection_fasttext_get_nlangs(rspamd_fasttext_predict_result_t ud);
+/**
+ * Get language from fasttext result
+ * @param res
+ * @return
+ */
+const char *rspamd_lang_detection_fasttext_get_lang(rspamd_fasttext_predict_result_t res, unsigned int idx);
+
+/**
+ * Get probability from fasttext result
+ * @param res
+ * @return
+ */
+float rspamd_lang_detection_fasttext_get_prob(rspamd_fasttext_predict_result_t res, unsigned int idx);
+
+/**
+ * Destroy fasttext result
+ * @param res
+ */
+void rspamd_fasttext_predict_result_destroy(rspamd_fasttext_predict_result_t res);
+
+/**
+ * Destroy fasttext language detector
+ */
+void rspamd_lang_detection_fasttext_destroy(void *ud);
+
+
+G_END_DECLS
+#endif /* RSPAMD_LANG_DETECTION_FASTTEXT_H */
diff --git a/src/libmime/message.c b/src/libmime/message.c
new file mode 100644
index 0000000..3acc935
--- /dev/null
+++ b/src/libmime/message.c
@@ -0,0 +1,1732 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "util.h"
+#include "rspamd.h"
+#include "message.h"
+#include "libserver/html/html.h"
+#include "images.h"
+#include "archives.h"
+#include "tokenizers/tokenizers.h"
+#include "smtp_parsers.h"
+#include "mime_parser.h"
+#include "mime_encoding.h"
+#include "lang_detection.h"
+#include "libutil/multipattern.h"
+#include "libserver/mempool_vars_internal.h"
+
+#ifdef WITH_SNOWBALL
+#include "libstemmer.h"
+#endif
+
+#include <math.h>
+#include <unicode/uchar.h>
+#include "sodium.h"
+#include "libserver/cfg_file_private.h"
+#include "lua/lua_common.h"
+#include "contrib/uthash/utlist.h"
+#include "contrib/t1ha/t1ha.h"
+#include "received.h"
+
+#define GTUBE_SYMBOL "GTUBE"
+
+#define SET_PART_RAW(part) ((part)->flags &= ~RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+#define SET_PART_UTF(part) ((part)->flags |= RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+
+static const gchar gtube_pattern_reject[] = "XJS*C4JDBQADN1.NSBN3*2IDNEN*"
+											"GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X";
+static const gchar gtube_pattern_add_header[] = "YJS*C4JDBQADN1.NSBN3*2IDNEN*"
+												"GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X";
+static const gchar gtube_pattern_rewrite_subject[] = "ZJS*C4JDBQADN1.NSBN3*2IDNEN*"
+													 "GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X";
+static const gchar gtube_pattern_no_action[] = "AJS*C4JDBQADN1.NSBN3*2IDNEN*"
+											   "GTUBE-STANDARD-ANTI-UBE-TEST-EMAIL*C.34X";
+struct rspamd_multipattern *gtube_matcher = NULL;
+static const guint64 words_hash_seed = 0xdeadbabe;
+
+static void
+free_byte_array_callback(void *pointer)
+{
+	GByteArray *arr = (GByteArray *) pointer;
+	g_byte_array_free(arr, TRUE);
+}
+
+static void
+rspamd_mime_part_extract_words(struct rspamd_task *task,
+							   struct rspamd_mime_text_part *part)
+{
+	rspamd_stat_token_t *w;
+	guint i, total_len = 0, short_len = 0;
+
+	if (part->utf_words) {
+		rspamd_stem_words(part->utf_words, task->task_pool, part->language,
+						  task->lang_det);
+
+		for (i = 0; i < part->utf_words->len; i++) {
+			guint64 h;
+
+			w = &g_array_index(part->utf_words, rspamd_stat_token_t, i);
+
+			if (w->stemmed.len > 0) {
+				/*
+				 * We use static hash seed if we would want to use that in shingles
+				 * computation in future
+				 */
+				h = rspamd_cryptobox_fast_hash_specific(
+					RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT,
+					w->stemmed.begin, w->stemmed.len, words_hash_seed);
+				g_array_append_val(part->normalized_hashes, h);
+				total_len += w->stemmed.len;
+
+				if (w->stemmed.len <= 3) {
+					short_len++;
+				}
+
+				if (w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT &&
+					!(w->flags & RSPAMD_STAT_TOKEN_FLAG_SKIPPED)) {
+					part->nwords++;
+				}
+			}
+
+			if (w->flags & (RSPAMD_STAT_TOKEN_FLAG_BROKEN_UNICODE |
+							RSPAMD_STAT_TOKEN_FLAG_NORMALISED |
+							RSPAMD_STAT_TOKEN_FLAG_INVISIBLE_SPACES)) {
+				task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
+			}
+		}
+
+		if (part->utf_words->len) {
+			gdouble *avg_len_p, *short_len_p;
+
+			avg_len_p = rspamd_mempool_get_variable(task->task_pool,
+													RSPAMD_MEMPOOL_AVG_WORDS_LEN);
+
+			if (avg_len_p == NULL) {
+				avg_len_p = rspamd_mempool_alloc(task->task_pool,
+												 sizeof(double));
+				*avg_len_p = total_len;
+				rspamd_mempool_set_variable(task->task_pool,
+											RSPAMD_MEMPOOL_AVG_WORDS_LEN, avg_len_p, NULL);
+			}
+			else {
+				*avg_len_p += total_len;
+			}
+
+			short_len_p = rspamd_mempool_get_variable(task->task_pool,
+													  RSPAMD_MEMPOOL_SHORT_WORDS_CNT);
+
+			if (short_len_p == NULL) {
+				short_len_p = rspamd_mempool_alloc(task->task_pool,
+												   sizeof(double));
+				*short_len_p = short_len;
+				rspamd_mempool_set_variable(task->task_pool,
+											RSPAMD_MEMPOOL_SHORT_WORDS_CNT, avg_len_p, NULL);
+			}
+			else {
+				*short_len_p += short_len;
+			}
+		}
+	}
+}
+
+static void
+rspamd_mime_part_create_words(struct rspamd_task *task,
+							  struct rspamd_mime_text_part *part)
+{
+	enum rspamd_tokenize_type tok_type;
+
+	if (IS_TEXT_PART_UTF(part)) {
+
+#if U_ICU_VERSION_MAJOR_NUM < 50
+		/* Hack to prevent hang with Thai in old libicu */
+		const gchar *p = part->utf_stripped_content->data, *end;
+		guint i = 0;
+		end = p + part->utf_stripped_content->len;
+		gint32 uc, sc;
+
+		tok_type = RSPAMD_TOKENIZE_UTF;
+
+		while (p + i < end) {
+			U8_NEXT(p, i, part->utf_stripped_content->len, uc);
+
+			if (((gint32) uc) < 0) {
+				tok_type = RSPAMD_TOKENIZE_RAW;
+				break;
+			}
+
+			if (u_isalpha(uc)) {
+				sc = ublock_getCode(uc);
+
+				if (sc == UBLOCK_THAI) {
+					msg_info_task("enable workaround for Thai characters for old libicu");
+					tok_type = RSPAMD_TOKENIZE_RAW;
+					break;
+				}
+			}
+		}
+#else
+		tok_type = RSPAMD_TOKENIZE_UTF;
+#endif
+	}
+	else {
+		tok_type = RSPAMD_TOKENIZE_RAW;
+	}
+
+	part->utf_words = rspamd_tokenize_text(
+		part->utf_stripped_content->data,
+		part->utf_stripped_content->len,
+		&part->utf_stripped_text,
+		tok_type, task->cfg,
+		part->exceptions,
+		NULL,
+		NULL,
+		task->task_pool);
+
+
+	if (part->utf_words) {
+		part->normalized_hashes = g_array_sized_new(FALSE, FALSE,
+													sizeof(guint64), part->utf_words->len);
+		rspamd_normalize_words(part->utf_words, task->task_pool);
+	}
+}
+
+static void
+rspamd_mime_part_detect_language(struct rspamd_task *task,
+								 struct rspamd_mime_text_part *part)
+{
+	struct rspamd_lang_detector_res *lang;
+
+	if (!IS_TEXT_PART_EMPTY(part) && part->utf_words && part->utf_words->len > 0 &&
+		task->lang_det) {
+		if (rspamd_language_detector_detect(task, task->lang_det, part)) {
+			lang = g_ptr_array_index(part->languages, 0);
+			part->language = lang->lang;
+
+			msg_info_task("detected part language: %s", part->language);
+		}
+		else {
+			part->language = "en"; /* Safe fallback */
+		}
+	}
+}
+
+static void
+rspamd_strip_newlines_parse(struct rspamd_task *task,
+							const gchar *begin, const gchar *pe,
+							struct rspamd_mime_text_part *part)
+{
+	const gchar *p = begin, *c = begin;
+	gboolean crlf_added = FALSE, is_utf = IS_TEXT_PART_UTF(part);
+	gboolean url_open_bracket = FALSE;
+	UChar32 uc;
+
+	enum {
+		normal_char,
+		seen_cr,
+		seen_lf,
+	} state = normal_char;
+
+	while (p < pe) {
+		if (U8_IS_LEAD(*p) && is_utf) {
+			gint32 off = p - begin;
+			U8_NEXT(begin, off, pe - begin, uc);
+
+			if (uc != -1) {
+				while (p < pe && off < (pe - begin)) {
+					if (IS_ZERO_WIDTH_SPACE(uc)) {
+						/* Invisible space ! */
+						task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
+						part->spaces++;
+
+						if (p > c) {
+							g_byte_array_append(part->utf_stripped_content,
+												(const guint8 *) c, p - c);
+							c = begin + off;
+							p = c;
+						}
+
+						U8_NEXT(begin, off, pe - begin, uc);
+
+						if (!IS_ZERO_WIDTH_SPACE(uc)) {
+							break;
+						}
+
+						part->double_spaces++;
+						p = begin + off;
+						c = p;
+					}
+					else {
+						break;
+					}
+				}
+			}
+		}
+
+		if (G_UNLIKELY(p >= pe)) {
+			/*
+			 * This is reached when there is a utf8 part and we
+			 * have zero width spaces at the end of the text
+			 * So we just check overflow and refuse to access *p if it is
+			 * after our real content.
+			 */
+			break;
+		}
+		else if (*p == '\r') {
+			switch (state) {
+			case normal_char:
+				state = seen_cr;
+				if (p > c) {
+					g_byte_array_append(part->utf_stripped_content,
+										(const guint8 *) c, p - c);
+				}
+
+				crlf_added = FALSE;
+				c = p + 1;
+				break;
+			case seen_cr:
+				/* Double \r\r */
+				if (!crlf_added) {
+					g_byte_array_append(part->utf_stripped_content,
+										(const guint8 *) " ", 1);
+					crlf_added = TRUE;
+					g_ptr_array_add(part->newlines,
+									(((gpointer) (goffset) (part->utf_stripped_content->len))));
+				}
+
+				part->nlines++;
+				part->empty_lines++;
+				c = p + 1;
+				break;
+			case seen_lf:
+				/* Likely \r\n\r...*/
+				state = seen_cr;
+				c = p + 1;
+				break;
+			}
+
+			url_open_bracket = FALSE;
+
+			p++;
+		}
+		else if (*p == '\n') {
+			switch (state) {
+			case normal_char:
+				state = seen_lf;
+
+				if (p > c) {
+					g_byte_array_append(part->utf_stripped_content,
+										(const guint8 *) c, p - c);
+				}
+
+				c = p + 1;
+
+				if (IS_TEXT_PART_HTML(part) || !url_open_bracket) {
+					g_byte_array_append(part->utf_stripped_content,
+										(const guint8 *) " ", 1);
+					g_ptr_array_add(part->newlines,
+									(((gpointer) (goffset) (part->utf_stripped_content->len))));
+					crlf_added = TRUE;
+				}
+				else {
+					crlf_added = FALSE;
+				}
+
+				break;
+			case seen_cr:
+				/* \r\n */
+				if (!crlf_added) {
+					if (IS_TEXT_PART_HTML(part) || !url_open_bracket) {
+						g_byte_array_append(part->utf_stripped_content,
+											(const guint8 *) " ", 1);
+						crlf_added = TRUE;
+					}
+
+					g_ptr_array_add(part->newlines,
+									(((gpointer) (goffset) (part->utf_stripped_content->len))));
+				}
+
+				c = p + 1;
+				state = seen_lf;
+
+				break;
+			case seen_lf:
+				/* Double \n\n */
+				if (!crlf_added) {
+					g_byte_array_append(part->utf_stripped_content,
+										(const guint8 *) " ", 1);
+					crlf_added = TRUE;
+					g_ptr_array_add(part->newlines,
+									(((gpointer) (goffset) (part->utf_stripped_content->len))));
+				}
+
+				part->nlines++;
+				part->empty_lines++;
+
+				c = p + 1;
+				break;
+			}
+			url_open_bracket = FALSE;
+
+			p++;
+		}
+		else {
+			if ((*p) == '<') {
+				url_open_bracket = TRUE;
+			}
+			else if ((*p) == '>') {
+				url_open_bracket = FALSE;
+			}
+
+			switch (state) {
+			case normal_char:
+				if (*p == ' ') {
+					part->spaces++;
+
+					if (p > begin && *(p - 1) == ' ') {
+						part->double_spaces++;
+					}
+				}
+				else {
+					part->non_spaces++;
+
+					if ((*p) & 0x80) {
+						part->non_ascii_chars++;
+					}
+					else {
+						if (g_ascii_isupper(*p)) {
+							part->capital_letters++;
+						}
+						else if (g_ascii_isdigit(*p)) {
+							part->numeric_characters++;
+						}
+
+						part->ascii_chars++;
+					}
+				}
+				break;
+			case seen_cr:
+			case seen_lf:
+				part->nlines++;
+
+				if (!crlf_added) {
+					g_ptr_array_add(part->newlines,
+									(((gpointer) (goffset) (part->utf_stripped_content->len))));
+				}
+
+				/* Skip initial spaces */
+				if (*p == ' ') {
+					if (!crlf_added) {
+						g_byte_array_append(part->utf_stripped_content,
+											(const guint8 *) " ", 1);
+					}
+
+					while (p < pe && *p == ' ') {
+						p++;
+						c++;
+						part->spaces++;
+					}
+
+					if (p < pe && (*p == '\r' || *p == '\n')) {
+						part->empty_lines++;
+					}
+				}
+
+				state = normal_char;
+				continue;
+			}
+
+			p++;
+		}
+	}
+
+	/* Leftover */
+	if (p > c) {
+		if (p > pe) {
+			p = pe;
+		}
+
+		switch (state) {
+		case normal_char:
+			g_byte_array_append(part->utf_stripped_content,
+								(const guint8 *) c, p - c);
+
+			while (c < p) {
+				if (*c == ' ') {
+					part->spaces++;
+
+					if (c > begin && *(c - 1) == ' ') {
+						part->double_spaces++;
+					}
+				}
+				else {
+					part->non_spaces++;
+
+					if ((*c) & 0x80) {
+						part->non_ascii_chars++;
+					}
+					else {
+						part->ascii_chars++;
+					}
+				}
+
+				c++;
+			}
+			break;
+		default:
+
+			if (!crlf_added) {
+				g_byte_array_append(part->utf_stripped_content,
+									(const guint8 *) " ", 1);
+				g_ptr_array_add(part->newlines,
+								(((gpointer) (goffset) (part->utf_stripped_content->len))));
+			}
+
+			part->nlines++;
+			break;
+		}
+	}
+}
+
+static void
+rspamd_u_text_dtor(void *p)
+{
+	utext_close((UText *) p);
+}
+
+static void
+rspamd_normalize_text_part(struct rspamd_task *task,
+						   struct rspamd_mime_text_part *part)
+{
+	const gchar *p, *end;
+	guint i;
+	goffset off;
+	struct rspamd_process_exception *ex;
+	UErrorCode uc_err = U_ZERO_ERROR;
+
+	part->newlines = g_ptr_array_sized_new(128);
+
+	if (IS_TEXT_PART_EMPTY(part)) {
+		part->utf_stripped_content = g_byte_array_new();
+	}
+	else {
+		part->utf_stripped_content = g_byte_array_sized_new(part->utf_content.len);
+
+		p = (const gchar *) part->utf_content.begin;
+		end = p + part->utf_content.len;
+
+		rspamd_strip_newlines_parse(task, p, end, part);
+
+		for (i = 0; i < part->newlines->len; i++) {
+			ex = rspamd_mempool_alloc(task->task_pool, sizeof(*ex));
+			off = (goffset) g_ptr_array_index(part->newlines, i);
+			g_ptr_array_index(part->newlines, i) = (gpointer) (goffset) (part->utf_stripped_content->data + off);
+			ex->pos = off;
+			ex->len = 0;
+			ex->type = RSPAMD_EXCEPTION_NEWLINE;
+			part->exceptions = g_list_prepend(part->exceptions, ex);
+		}
+	}
+
+	if (IS_TEXT_PART_UTF(part)) {
+		utext_openUTF8(&part->utf_stripped_text,
+					   part->utf_stripped_content->data,
+					   part->utf_stripped_content->len,
+					   &uc_err);
+
+		if (!U_SUCCESS(uc_err)) {
+			msg_warn_task("cannot open text from utf content");
+			/* Probably, should be an assertion */
+		}
+		else {
+			rspamd_mempool_add_destructor(task->task_pool,
+										  rspamd_u_text_dtor,
+										  &part->utf_stripped_text);
+		}
+	}
+
+	rspamd_mempool_add_destructor(task->task_pool,
+								  (rspamd_mempool_destruct_t) free_byte_array_callback,
+								  part->utf_stripped_content);
+	rspamd_mempool_notify_alloc(task->task_pool,
+								part->utf_stripped_content->len);
+	rspamd_mempool_add_destructor(task->task_pool,
+								  (rspamd_mempool_destruct_t) rspamd_ptr_array_free_hard,
+								  part->newlines);
+}
+
+#define MIN3(a, b, c) ((a) < (b) ? ((a) < (c) ? (a) : (c)) : ((b) < (c) ? (b) : (c)))
+
+static guint
+rspamd_words_levenshtein_distance(struct rspamd_task *task,
+								  GArray *w1, GArray *w2)
+{
+	guint s1len, s2len, x, y, lastdiag, olddiag;
+	guint *column, ret;
+	guint64 h1, h2;
+	gint eq;
+	static const guint max_words = 8192;
+
+	s1len = w1->len;
+	s2len = w2->len;
+
+	if (s1len + s2len > max_words) {
+		msg_info_task("cannot direct compare multipart/alternative parts with more than %ud words in total: "
+					  "(%ud words in one part and %ud in another)",
+					  max_words, s1len, s2len);
+
+		/* Use approximate comparison of number of words */
+		if (s1len > s2len) {
+			return s1len - s2len;
+		}
+		else {
+			return s2len - s1len;
+		}
+	}
+
+	column = g_malloc0((s1len + 1) * sizeof(guint));
+
+	for (y = 1; y <= s1len; y++) {
+		column[y] = y;
+	}
+
+	for (x = 1; x <= s2len; x++) {
+		column[0] = x;
+
+		for (y = 1, lastdiag = x - 1; y <= s1len; y++) {
+			olddiag = column[y];
+			h1 = g_array_index(w1, guint64, y - 1);
+			h2 = g_array_index(w2, guint64, x - 1);
+			eq = (h1 == h2) ? 1 : 0;
+			/*
+			 * Cost of replacement is twice higher than cost of add/delete
+			 * to calculate percentage properly
+			 */
+			column[y] = MIN3(column[y] + 1, column[y - 1] + 1,
+							 lastdiag + (eq * 2));
+			lastdiag = olddiag;
+		}
+	}
+
+	ret = column[s1len];
+	g_free(column);
+
+	return ret;
+}
+
+static gint
+rspamd_multipattern_gtube_cb(struct rspamd_multipattern *mp,
+							 guint strnum,
+							 gint match_start,
+							 gint match_pos,
+							 const gchar *text,
+							 gsize len,
+							 void *context)
+{
+	struct rspamd_task *task = (struct rspamd_task *) context;
+
+	if (strnum > 0) {
+		if (task->cfg->gtube_patterns_policy == RSPAMD_GTUBE_ALL) {
+			return strnum + 1;
+		}
+
+		return 0;
+	}
+
+	return strnum + 1; /* To distinguish from zero */
+}
+
+static enum rspamd_action_type
+rspamd_check_gtube(struct rspamd_task *task, struct rspamd_mime_text_part *part)
+{
+	static const gsize max_check_size = 8 * 1024;
+	gint ret;
+	enum rspamd_action_type act = METRIC_ACTION_NOACTION;
+	enum rspamd_gtube_patterns_policy policy = task->cfg ? task->cfg->gtube_patterns_policy : RSPAMD_GTUBE_REJECT;
+	g_assert(part != NULL);
+
+	if (gtube_matcher == NULL && policy != RSPAMD_GTUBE_DISABLED) {
+		gtube_matcher = rspamd_multipattern_create(RSPAMD_MULTIPATTERN_DEFAULT);
+
+		rspamd_multipattern_add_pattern(gtube_matcher,
+										gtube_pattern_reject,
+										RSPAMD_MULTIPATTERN_DEFAULT);
+		rspamd_multipattern_add_pattern(gtube_matcher,
+										gtube_pattern_add_header,
+										RSPAMD_MULTIPATTERN_DEFAULT);
+		rspamd_multipattern_add_pattern(gtube_matcher,
+										gtube_pattern_rewrite_subject,
+										RSPAMD_MULTIPATTERN_DEFAULT);
+		rspamd_multipattern_add_pattern(gtube_matcher,
+										gtube_pattern_no_action,
+										RSPAMD_MULTIPATTERN_DEFAULT);
+
+		GError *err = NULL;
+		rspamd_multipattern_compile(gtube_matcher, &err);
+
+		if (err != NULL) {
+			/* It will be expensive, but I don't care, still better than to abort */
+			msg_err("cannot compile gtube matcher: %s", err->message);
+			g_error_free(err);
+		}
+	}
+
+	if (part->utf_content.len >= sizeof(gtube_pattern_reject) &&
+		part->utf_content.len <= max_check_size &&
+		policy != RSPAMD_GTUBE_DISABLED) {
+		if ((ret = rspamd_multipattern_lookup(gtube_matcher, part->utf_content.begin,
+											  part->utf_content.len,
+											  rspamd_multipattern_gtube_cb, task, NULL)) > 0) {
+
+			switch (ret) {
+			case 1:
+				act = METRIC_ACTION_REJECT;
+				break;
+			case 2:
+				act = METRIC_ACTION_ADD_HEADER;
+				break;
+			case 3:
+				act = METRIC_ACTION_REWRITE_SUBJECT;
+				break;
+			case 4:
+				act = METRIC_ACTION_NOACTION;
+				break;
+			}
+
+			if (ret != 0) {
+				task->flags |= RSPAMD_TASK_FLAG_SKIP;
+				task->flags |= RSPAMD_TASK_FLAG_GTUBE;
+				msg_info_task(
+					"gtube %s pattern has been found in part of length %uz",
+					rspamd_action_to_str(act),
+					part->utf_content.len);
+			}
+		}
+	}
+
+	return act;
+}
+
+static gint
+exceptions_compare_func(gconstpointer a, gconstpointer b)
+{
+	const struct rspamd_process_exception *ea = a, *eb = b;
+
+	return ea->pos - eb->pos;
+}
+
+static gboolean
+rspamd_message_process_plain_text_part(struct rspamd_task *task,
+									   struct rspamd_mime_text_part *text_part)
+{
+	if (text_part->parsed.len == 0) {
+		text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_EMPTY;
+
+		return TRUE;
+	}
+
+	rspamd_mime_text_part_maybe_convert(task, text_part);
+
+	if (text_part->utf_raw_content != NULL) {
+		/* Just have the same content */
+		text_part->utf_content.begin = (const gchar *) text_part->utf_raw_content->data;
+		text_part->utf_content.len = text_part->utf_raw_content->len;
+	}
+	else {
+		/*
+		 * We ignore unconverted parts from now as it is dangerous
+		 * to treat them as text parts
+		 */
+		text_part->utf_content.begin = NULL;
+		text_part->utf_content.len = 0;
+
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static gboolean
+rspamd_message_process_html_text_part(struct rspamd_task *task,
+									  struct rspamd_mime_text_part *text_part,
+									  uint16_t *cur_url_order)
+{
+	text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_HTML;
+
+	if (text_part->parsed.len == 0) {
+		text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_EMPTY;
+
+		return TRUE;
+	}
+
+	rspamd_mime_text_part_maybe_convert(task, text_part);
+
+	if (text_part->utf_raw_content == NULL) {
+		return FALSE;
+	}
+
+
+	text_part->html = rspamd_html_process_part_full(
+		task,
+		text_part->utf_raw_content,
+		&text_part->exceptions,
+		MESSAGE_FIELD(task, urls),
+		text_part->mime_part->urls,
+		task->cfg ? task->cfg->enable_css_parser : true,
+		cur_url_order);
+	rspamd_html_get_parsed_content(text_part->html, &text_part->utf_content);
+
+	if (text_part->utf_content.len == 0) {
+		text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_EMPTY;
+	}
+
+	return TRUE;
+}
+
+enum rspamd_message_part_is_text_result {
+	RSPAMD_MESSAGE_PART_IS_TEXT_PLAIN = 0,
+	RSPAMD_MESSAGE_PART_IS_TEXT_HTML,
+	RSPAMD_MESSAGE_PART_IS_NOT_TEXT
+};
+
+static enum rspamd_message_part_is_text_result
+rspamd_message_part_can_be_parsed_as_text(struct rspamd_task *task,
+										  struct rspamd_mime_part *mime_part)
+{
+	enum rspamd_message_part_is_text_result res = RSPAMD_MESSAGE_PART_IS_NOT_TEXT;
+
+	if ((mime_part->ct && (mime_part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) ||
+		(mime_part->detected_type && strcmp(mime_part->detected_type, "text") == 0)) {
+
+		res = RSPAMD_MESSAGE_PART_IS_TEXT_PLAIN;
+		rspamd_ftok_t html_tok, xhtml_tok;
+
+		html_tok.begin = "html";
+		html_tok.len = 4;
+		xhtml_tok.begin = "xhtml";
+		xhtml_tok.len = 5;
+
+		if (rspamd_ftok_casecmp(&mime_part->ct->subtype, &html_tok) == 0 ||
+			rspamd_ftok_casecmp(&mime_part->ct->subtype, &xhtml_tok) == 0 ||
+			(mime_part->detected_ext &&
+			 strcmp(mime_part->detected_ext, "html") == 0)) {
+			res = RSPAMD_MESSAGE_PART_IS_TEXT_HTML;
+		}
+	}
+
+	/* Skip attachments */
+	if (res != RSPAMD_MESSAGE_PART_IS_NOT_TEXT &&
+		(mime_part->cd && mime_part->cd->type == RSPAMD_CT_ATTACHMENT)) {
+		if (!task->cfg->check_text_attachements) {
+			debug_task("skip attachments for checking as text parts");
+			return RSPAMD_MESSAGE_PART_IS_NOT_TEXT;
+		}
+	}
+
+	return res;
+}
+
+static gboolean
+rspamd_message_process_text_part_maybe(struct rspamd_task *task,
+									   struct rspamd_mime_part *mime_part,
+									   enum rspamd_message_part_is_text_result is_text,
+									   uint16_t *cur_url_order)
+{
+	struct rspamd_mime_text_part *text_part;
+	guint flags = 0;
+	enum rspamd_action_type act;
+
+	/* Skip attachments */
+	if ((mime_part->cd && mime_part->cd->type == RSPAMD_CT_ATTACHMENT)) {
+		flags |= RSPAMD_MIME_TEXT_PART_ATTACHMENT;
+	}
+
+	text_part = rspamd_mempool_alloc0(task->task_pool,
+									  sizeof(struct rspamd_mime_text_part));
+	text_part->mime_part = mime_part;
+	text_part->raw.begin = mime_part->raw_data.begin;
+	text_part->raw.len = mime_part->raw_data.len;
+	text_part->parsed.begin = mime_part->parsed_data.begin;
+	text_part->parsed.len = mime_part->parsed_data.len;
+	text_part->utf_stripped_text = (UText) UTEXT_INITIALIZER;
+	text_part->flags |= flags;
+
+	if (is_text == RSPAMD_MESSAGE_PART_IS_TEXT_HTML) {
+		if (!rspamd_message_process_html_text_part(task, text_part, cur_url_order)) {
+			return FALSE;
+		}
+	}
+	else {
+		if (!rspamd_message_process_plain_text_part(task, text_part)) {
+			return FALSE;
+		}
+	}
+
+	g_ptr_array_add(MESSAGE_FIELD(task, text_parts), text_part);
+	mime_part->part_type = RSPAMD_MIME_PART_TEXT;
+	mime_part->specific.txt = text_part;
+
+	act = rspamd_check_gtube(task, text_part);
+	if (act != METRIC_ACTION_NOACTION) {
+		struct rspamd_action *action;
+		gdouble score = NAN;
+
+		action = rspamd_config_get_action_by_type(task->cfg, act);
+
+		if (action) {
+			score = action->threshold;
+
+			rspamd_add_passthrough_result(task, action,
+										  RSPAMD_PASSTHROUGH_CRITICAL,
+										  score, "Gtube pattern",
+										  "GTUBE", 0, NULL);
+		}
+
+		rspamd_task_insert_result(task, GTUBE_SYMBOL, 0, NULL);
+
+		return TRUE;
+	}
+
+	/* Post process part */
+	rspamd_normalize_text_part(task, text_part);
+
+	if (!IS_TEXT_PART_HTML(text_part)) {
+		if (mime_part->parent_part) {
+			struct rspamd_mime_part *parent = mime_part->parent_part;
+
+			if (IS_PART_MULTIPART(parent) && parent->specific.mp->children->len == 2) {
+				/*
+				 * Use strict extraction mode: we will extract missing urls from
+				 * an html part if needed
+				 */
+				rspamd_url_text_extract(task->task_pool, task, text_part, cur_url_order,
+										RSPAMD_URL_FIND_STRICT);
+			}
+			else {
+				/*
+				 * Fall back to full text extraction using TLD patterns
+				 */
+				rspamd_url_text_extract(task->task_pool, task, text_part, cur_url_order,
+										RSPAMD_URL_FIND_ALL);
+			}
+		}
+		else {
+			/*
+			 * Fall back to full text extraction using TLD patterns
+			*/
+			rspamd_url_text_extract(task->task_pool, task, text_part, cur_url_order,
+									RSPAMD_URL_FIND_ALL);
+		}
+	}
+	else {
+		rspamd_url_text_extract(task->task_pool, task, text_part, cur_url_order,
+								RSPAMD_URL_FIND_STRICT);
+	}
+
+	if (text_part->exceptions) {
+		text_part->exceptions = g_list_sort(text_part->exceptions,
+											exceptions_compare_func);
+		rspamd_mempool_add_destructor(task->task_pool,
+									  (rspamd_mempool_destruct_t) g_list_free,
+									  text_part->exceptions);
+	}
+
+	rspamd_mime_part_create_words(task, text_part);
+
+	return TRUE;
+}
+
+/* Creates message from various data using libmagic to detect type */
+static void
+rspamd_message_from_data(struct rspamd_task *task, const guchar *start,
+						 gsize len)
+{
+	struct rspamd_content_type *ct = NULL;
+	struct rspamd_mime_part *part;
+	const char *mb = "application/octet-stream";
+	gchar *mid;
+	rspamd_ftok_t srch, *tok;
+	gchar cdbuf[1024];
+
+	g_assert(start != NULL);
+
+	part = rspamd_mempool_alloc0(task->task_pool, sizeof(*part));
+
+	part->raw_data.begin = start;
+	part->raw_data.len = len;
+	part->parsed_data.begin = start;
+	part->parsed_data.len = len;
+	part->part_number = MESSAGE_FIELD(task, parts)->len;
+	part->urls = g_ptr_array_new();
+	part->raw_headers = rspamd_message_headers_new();
+	part->headers_order = NULL;
+
+	tok = rspamd_task_get_request_header(task, "Content-Type");
+
+	if (tok) {
+		/* We have Content-Type defined */
+		ct = rspamd_content_type_parse(tok->begin, tok->len,
+									   task->task_pool);
+		part->ct = ct;
+	}
+	else if (task->cfg && task->cfg->libs_ctx) {
+		lua_State *L = task->cfg->lua_state;
+
+		if (rspamd_lua_require_function(L,
+										"lua_magic", "detect_mime_part")) {
+
+			struct rspamd_mime_part **pmime;
+			struct rspamd_task **ptask;
+
+			pmime = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+			rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+			*pmime = part;
+			ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+			rspamd_lua_setclass(L, "rspamd{task}", -1);
+			*ptask = task;
+
+			if (lua_pcall(L, 2, 2, 0) != 0) {
+				msg_err_task("cannot detect type: %s", lua_tostring(L, -1));
+			}
+			else {
+				if (lua_istable(L, -1)) {
+					lua_pushstring(L, "ct");
+					lua_gettable(L, -2);
+
+					if (lua_isstring(L, -1)) {
+						mb = rspamd_mempool_strdup(task->task_pool,
+												   lua_tostring(L, -1));
+					}
+				}
+			}
+
+			lua_settop(L, 0);
+		}
+		else {
+			msg_err_task("cannot require lua_magic.detect_mime_part");
+		}
+
+		if (mb) {
+			srch.begin = mb;
+			srch.len = strlen(mb);
+			ct = rspamd_content_type_parse(srch.begin, srch.len,
+										   task->task_pool);
+
+			if (!part->ct) {
+				msg_info_task("construct fake mime of type: %s", mb);
+				part->ct = ct;
+			}
+			else {
+				/* Check sanity */
+				if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) {
+					RSPAMD_FTOK_FROM_STR(&srch, "application");
+
+					if (rspamd_ftok_cmp(&ct->type, &srch) == 0) {
+						msg_info_task("construct fake mime of type: %s", mb);
+						part->ct = ct;
+					}
+				}
+				else {
+					msg_info_task("construct fake mime of type: %T/%T, detected %s",
+								  &part->ct->type, &part->ct->subtype, mb);
+				}
+			}
+
+			part->detected_ct = ct;
+		}
+	}
+
+
+	tok = rspamd_task_get_request_header(task, "Filename");
+
+	if (tok) {
+		rspamd_snprintf(cdbuf, sizeof(cdbuf), "inline; filename=\"%T\"", tok);
+	}
+	else {
+		rspamd_snprintf(cdbuf, sizeof(cdbuf), "inline");
+	}
+
+	part->cd = rspamd_content_disposition_parse(cdbuf, strlen(cdbuf),
+												task->task_pool);
+
+	g_ptr_array_add(MESSAGE_FIELD(task, parts), part);
+	rspamd_mime_parser_calc_digest(part);
+
+	/* Generate message ID */
+	mid = rspamd_mime_message_id_generate("localhost.localdomain");
+	rspamd_mempool_add_destructor(task->task_pool,
+								  (rspamd_mempool_destruct_t) g_free, mid);
+	MESSAGE_FIELD(task, message_id) = mid;
+	task->queue_id = mid;
+}
+
+static void
+rspamd_message_dtor(struct rspamd_message *msg)
+{
+	guint i;
+	struct rspamd_mime_part *p;
+	struct rspamd_mime_text_part *tp;
+
+
+	PTR_ARRAY_FOREACH(msg->parts, i, p)
+	{
+		if (p->raw_headers) {
+			rspamd_message_headers_unref(p->raw_headers);
+		}
+
+		if (IS_PART_MULTIPART(p)) {
+			if (p->specific.mp->children) {
+				g_ptr_array_free(p->specific.mp->children, TRUE);
+			}
+		}
+
+		if (p->part_type == RSPAMD_MIME_PART_CUSTOM_LUA &&
+			p->specific.lua_specific.cbref != -1) {
+			luaL_unref(msg->task->cfg->lua_state,
+					   LUA_REGISTRYINDEX,
+					   p->specific.lua_specific.cbref);
+		}
+
+		if (p->urls) {
+			g_ptr_array_unref(p->urls);
+		}
+	}
+
+	PTR_ARRAY_FOREACH(msg->text_parts, i, tp)
+	{
+		if (tp->utf_words) {
+			g_array_free(tp->utf_words, TRUE);
+		}
+		if (tp->normalized_hashes) {
+			g_array_free(tp->normalized_hashes, TRUE);
+		}
+		if (tp->languages) {
+			g_ptr_array_unref(tp->languages);
+		}
+	}
+
+	rspamd_message_headers_unref(msg->raw_headers);
+
+	g_ptr_array_unref(msg->text_parts);
+	g_ptr_array_unref(msg->parts);
+
+	kh_destroy(rspamd_url_hash, msg->urls);
+}
+
+struct rspamd_message *
+rspamd_message_new(struct rspamd_task *task)
+{
+	struct rspamd_message *msg;
+
+	msg = rspamd_mempool_alloc0(task->task_pool, sizeof(*msg));
+
+	msg->raw_headers = rspamd_message_headers_new();
+	msg->urls = kh_init(rspamd_url_hash);
+	msg->parts = g_ptr_array_sized_new(4);
+	msg->text_parts = g_ptr_array_sized_new(2);
+	msg->task = task;
+
+	REF_INIT_RETAIN(msg, rspamd_message_dtor);
+
+	return msg;
+}
+
+gboolean
+rspamd_message_parse(struct rspamd_task *task)
+{
+	const gchar *p;
+	gsize len;
+	guint i;
+	GError *err = NULL;
+	guint64 n[2], seed;
+
+	if (RSPAMD_TASK_IS_EMPTY(task)) {
+		/* Don't do anything with empty task */
+		task->flags |= RSPAMD_TASK_FLAG_SKIP_PROCESS;
+		return TRUE;
+	}
+
+	p = task->msg.begin;
+	len = task->msg.len;
+
+	/* Skip any space characters to avoid some bad messages to be unparsed */
+	while (len > 0 && g_ascii_isspace(*p)) {
+		p++;
+		len--;
+	}
+
+	/*
+	 * Exim somehow uses mailbox format for messages being scanned:
+	 * From xxx@xxx.com Fri May 13 19:08:48 2016
+	 *
+	 * So we check if a task has this line to avoid possible issues
+	 */
+	if (len > sizeof("From ") - 1) {
+		if (memcmp(p, "From ", sizeof("From ") - 1) == 0) {
+			/* Skip to CRLF */
+			msg_info_task("mailbox input detected, enable workaround");
+			p += sizeof("From ") - 1;
+			len -= sizeof("From ") - 1;
+
+			while (len > 0 && *p != '\n') {
+				p++;
+				len--;
+			}
+			while (len > 0 && g_ascii_isspace(*p)) {
+				p++;
+				len--;
+			}
+		}
+	}
+
+	task->msg.begin = p;
+	task->msg.len = len;
+
+	/* Cleanup old message */
+	if (task->message) {
+		rspamd_message_unref(task->message);
+	}
+
+	task->message = rspamd_message_new(task);
+
+	if (task->flags & RSPAMD_TASK_FLAG_MIME) {
+		enum rspamd_mime_parse_error ret;
+
+		debug_task("construct mime parser from string length %d",
+				   (gint) task->msg.len);
+		ret = rspamd_mime_parse_task(task, &err);
+
+		switch (ret) {
+		case RSPAMD_MIME_PARSE_FATAL:
+			msg_err_task("cannot construct mime from stream: %e", err);
+
+			if (task->cfg && (!task->cfg->allow_raw_input)) {
+				msg_err_task("cannot construct mime from stream");
+				if (err) {
+					task->err = err;
+				}
+
+				return FALSE;
+			}
+			else {
+				task->flags &= ~RSPAMD_TASK_FLAG_MIME;
+				rspamd_message_from_data(task, p, len);
+			}
+			break;
+		case RSPAMD_MIME_PARSE_NESTING:
+			msg_warn_task("cannot construct full mime from stream: %e", err);
+			task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+			break;
+		case RSPAMD_MIME_PARSE_OK:
+		default:
+			break;
+		}
+
+		if (err) {
+			g_error_free(err);
+		}
+	}
+	else {
+		rspamd_message_from_data(task, p, len);
+	}
+
+
+	if (MESSAGE_FIELD(task, message_id) == NULL) {
+		MESSAGE_FIELD(task, message_id) = "undef";
+	}
+
+	debug_task("found %ud parts in message", MESSAGE_FIELD(task, parts)->len);
+	if (task->queue_id == NULL) {
+		task->queue_id = "undef";
+	}
+
+	rspamd_received_maybe_fix_task(task);
+
+	struct rspamd_mime_part *part;
+
+	/* Blake2b applied to string 'rspamd' */
+	static const guchar RSPAMD_ALIGNED(32) hash_key[] = {
+		0xef,
+		0x43,
+		0xae,
+		0x80,
+		0xcc,
+		0x8d,
+		0xc3,
+		0x4c,
+		0x6f,
+		0x1b,
+		0xd6,
+		0x18,
+		0x1b,
+		0xae,
+		0x87,
+		0x74,
+		0x0c,
+		0xca,
+		0xf7,
+		0x8e,
+		0x5f,
+		0x2e,
+		0x54,
+		0x32,
+		0xf6,
+		0x79,
+		0xb9,
+		0x27,
+		0x26,
+		0x96,
+		0x20,
+		0x92,
+		0x70,
+		0x07,
+		0x85,
+		0xeb,
+		0x83,
+		0xf7,
+		0x89,
+		0xe0,
+		0xd7,
+		0x32,
+		0x2a,
+		0xd2,
+		0x1a,
+		0x64,
+		0x41,
+		0xef,
+		0x49,
+		0xff,
+		0xc3,
+		0x8c,
+		0x54,
+		0xf9,
+		0x67,
+		0x74,
+		0x30,
+		0x1e,
+		0x70,
+		0x2e,
+		0xb7,
+		0x12,
+		0x09,
+		0xfe,
+	};
+
+	memcpy(&seed, hash_key, sizeof(seed));
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+	{
+		n[0] = t1ha2_atonce128(&n[1],
+							   part->digest, sizeof(part->digest),
+							   seed);
+
+		seed = n[0] ^ n[1];
+	}
+
+	memcpy(MESSAGE_FIELD(task, digest), n, sizeof(n));
+
+	if (MESSAGE_FIELD(task, subject)) {
+		p = MESSAGE_FIELD(task, subject);
+		len = strlen(p);
+		n[0] = t1ha2_atonce128(&n[1],
+							   p, len,
+							   seed);
+		memcpy(MESSAGE_FIELD(task, digest), n, sizeof(n));
+	}
+
+	if (task->queue_id) {
+		msg_info_task("loaded message; id: <%s>; queue-id: <%s>; size: %z; "
+					  "checksum: <%*xs>",
+					  MESSAGE_FIELD(task, message_id), task->queue_id, task->msg.len,
+					  (gint) sizeof(MESSAGE_FIELD(task, digest)), MESSAGE_FIELD(task, digest));
+	}
+	else {
+		msg_info_task("loaded message; id: <%s>; size: %z; "
+					  "checksum: <%*xs>",
+					  MESSAGE_FIELD(task, message_id), task->msg.len,
+					  (gint) sizeof(MESSAGE_FIELD(task, digest)), MESSAGE_FIELD(task, digest));
+	}
+
+	return TRUE;
+}
+
+
+/*
+ * A helper structure to store text parts positions, if it was C++, I could just use std::pair,
+ * but here I have to make it all manually, sigh...
+ */
+struct rspamd_mime_part_text_position {
+	unsigned pos;
+	enum rspamd_message_part_is_text_result res;
+};
+
+/* Place html parts first during analysis */
+static int
+rspamd_mime_text_part_position_compare_func(const void *v1, const void *v2)
+{
+	const struct rspamd_mime_part_text_position *p1 = (const struct rspamd_mime_part_text_position *) v1;
+	const struct rspamd_mime_part_text_position *p2 = (const struct rspamd_mime_part_text_position *) v2;
+
+	if (p1->res == p2->res) {
+		return (int) p2->pos - (int) p1->pos;
+	}
+	else {
+		if (p1->res == RSPAMD_MESSAGE_PART_IS_TEXT_HTML) {
+			return -1;
+		}
+		else {
+			return 1;
+		}
+	}
+}
+
+void rspamd_message_process(struct rspamd_task *task)
+{
+	guint i;
+	struct rspamd_mime_text_part *p1, *p2;
+	gdouble diff, *pdiff;
+	guint tw, *ptw, dw;
+	struct rspamd_mime_part *part;
+	lua_State *L = NULL;
+	gint magic_func_pos = -1, content_func_pos = -1, old_top = -1, funcs_top = -1;
+
+	if (task->cfg) {
+		L = task->cfg->lua_state;
+	}
+
+	rspamd_archives_process(task);
+
+	if (L) {
+		old_top = lua_gettop(L);
+	}
+
+	if (L && rspamd_lua_require_function(L,
+										 "lua_magic", "detect_mime_part")) {
+		magic_func_pos = lua_gettop(L);
+	}
+	else {
+		msg_err_task("cannot require lua_magic.detect_mime_part");
+	}
+
+	if (L && rspamd_lua_require_function(L,
+										 "lua_content", "maybe_process_mime_part")) {
+		content_func_pos = lua_gettop(L);
+	}
+	else {
+		msg_err_task("cannot require lua_content.maybe_process_mime_part");
+	}
+
+	if (L) {
+		funcs_top = lua_gettop(L);
+	}
+
+	GArray *detected_text_parts = g_array_sized_new(FALSE, FALSE, sizeof(struct rspamd_mime_part_text_position), 2);
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+	{
+		if (magic_func_pos != -1 && part->parsed_data.len > 0) {
+			struct rspamd_mime_part **pmime;
+			struct rspamd_task **ptask;
+
+			lua_pushcfunction(L, &rspamd_lua_traceback);
+			gint err_idx = lua_gettop(L);
+			lua_pushvalue(L, magic_func_pos);
+			pmime = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+			rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+			*pmime = part;
+			ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+			rspamd_lua_setclass(L, "rspamd{task}", -1);
+			*ptask = task;
+
+			if (lua_pcall(L, 2, 2, err_idx) != 0) {
+				msg_err_task("cannot detect type: %s", lua_tostring(L, -1));
+			}
+			else {
+				if (lua_istable(L, -1)) {
+					const gchar *mb;
+
+					/* First returned value */
+					part->detected_ext = rspamd_mempool_strdup(task->task_pool,
+															   lua_tostring(L, -2));
+
+					lua_pushstring(L, "ct");
+					lua_gettable(L, -2);
+
+					if (lua_isstring(L, -1)) {
+						mb = lua_tostring(L, -1);
+
+						if (mb) {
+							rspamd_ftok_t srch;
+
+							srch.begin = mb;
+							srch.len = strlen(mb);
+							part->detected_ct = rspamd_content_type_parse(srch.begin,
+																		  srch.len,
+																		  task->task_pool);
+						}
+					}
+
+					lua_pop(L, 1);
+
+					lua_pushstring(L, "type");
+					lua_gettable(L, -2);
+
+					if (lua_isstring(L, -1)) {
+						part->detected_type = rspamd_mempool_strdup(task->task_pool,
+																	lua_tostring(L, -1));
+					}
+
+					lua_pop(L, 1);
+
+					lua_pushstring(L, "no_text");
+					lua_gettable(L, -2);
+
+					if (lua_isboolean(L, -1)) {
+						if (!!lua_toboolean(L, -1)) {
+							part->flags |= RSPAMD_MIME_PART_NO_TEXT_EXTRACTION;
+						}
+					}
+
+					lua_pop(L, 1);
+				}
+			}
+
+			lua_settop(L, funcs_top);
+		}
+
+		/* Now detect content */
+		if (content_func_pos != -1 && part->parsed_data.len > 0 &&
+			part->part_type == RSPAMD_MIME_PART_UNDEFINED) {
+			struct rspamd_mime_part **pmime;
+			struct rspamd_task **ptask;
+
+			lua_pushcfunction(L, &rspamd_lua_traceback);
+			gint err_idx = lua_gettop(L);
+			lua_pushvalue(L, content_func_pos);
+			pmime = lua_newuserdata(L, sizeof(struct rspamd_mime_part *));
+			rspamd_lua_setclass(L, "rspamd{mimepart}", -1);
+			*pmime = part;
+			ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+			rspamd_lua_setclass(L, "rspamd{task}", -1);
+			*ptask = task;
+
+			if (lua_pcall(L, 2, 0, err_idx) != 0) {
+				msg_err_task("cannot detect content: %s", lua_tostring(L, -1));
+			}
+
+			lua_settop(L, funcs_top);
+		}
+
+		/* Try to detect image before checking for text */
+		rspamd_images_process_mime_part_maybe(task, part);
+
+		if (part->part_type == RSPAMD_MIME_PART_UNDEFINED &&
+			!(part->flags & RSPAMD_MIME_PART_NO_TEXT_EXTRACTION)) {
+			enum rspamd_message_part_is_text_result res = rspamd_message_part_can_be_parsed_as_text(task, part);
+
+			if (res != RSPAMD_MESSAGE_PART_IS_NOT_TEXT) {
+				struct rspamd_mime_part_text_position p = {
+					.pos = i,
+					.res = res};
+				g_array_append_val(detected_text_parts, p);
+			}
+		}
+	}
+
+	uint16_t cur_url_order = 0;
+	g_array_sort(detected_text_parts, rspamd_mime_text_part_position_compare_func);
+	/* One more iteration to process text parts in a more specific order */
+	for (i = 0; i < detected_text_parts->len; i++) {
+		part = g_ptr_array_index(MESSAGE_FIELD(task, parts),
+								 g_array_index(detected_text_parts, struct rspamd_mime_part_text_position, i).pos);
+		rspamd_message_process_text_part_maybe(task, part,
+											   g_array_index(detected_text_parts, struct rspamd_mime_part_text_position, i).res, &cur_url_order);
+	}
+
+	g_array_free(detected_text_parts, TRUE);
+
+	if (old_top != -1) {
+		lua_settop(L, old_top);
+	}
+
+	/* Parse urls inside Subject header */
+	if (MESSAGE_FIELD(task, subject)) {
+		rspamd_url_find_multiple(task->task_pool, MESSAGE_FIELD(task, subject),
+								 strlen(MESSAGE_FIELD(task, subject)),
+								 RSPAMD_URL_FIND_STRICT, NULL,
+								 rspamd_url_task_subject_callback,
+								 task);
+	}
+
+	/* Calculate average words length and number of short words */
+	struct rspamd_mime_text_part *text_part;
+	gdouble *var;
+	guint total_words = 0;
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, text_part)
+	{
+		if (!text_part->language) {
+			rspamd_mime_part_detect_language(task, text_part);
+		}
+
+		rspamd_mime_part_extract_words(task, text_part);
+
+		if (text_part->utf_words) {
+			total_words += text_part->nwords;
+		}
+	}
+
+	/* Calculate distance for 2-parts messages */
+	if (i == 2) {
+		p1 = g_ptr_array_index(MESSAGE_FIELD(task, text_parts), 0);
+		p2 = g_ptr_array_index(MESSAGE_FIELD(task, text_parts), 1);
+
+		/* First of all check parent object */
+		if (p1->mime_part->parent_part) {
+			rspamd_ftok_t srch;
+
+			srch.begin = "alternative";
+			srch.len = 11;
+
+			if (rspamd_ftok_cmp(&p1->mime_part->parent_part->ct->subtype, &srch) == 0) {
+				if (!IS_TEXT_PART_EMPTY(p1) && !IS_TEXT_PART_EMPTY(p2) &&
+					p1->normalized_hashes && p2->normalized_hashes) {
+					/*
+					 * We also detect language on one part and propagate it to
+					 * another one
+					 */
+					struct rspamd_mime_text_part *sel;
+
+					/* Prefer HTML as text part is not displayed normally */
+					if (IS_TEXT_PART_HTML(p1)) {
+						sel = p1;
+					}
+					else if (IS_TEXT_PART_HTML(p2)) {
+						sel = p2;
+					}
+					else {
+						if (p1->utf_content.len > p2->utf_content.len) {
+							sel = p1;
+						}
+						else {
+							sel = p2;
+						}
+					}
+
+					if (sel->language && sel->language[0]) {
+						/* Propagate language */
+						if (sel == p1) {
+							if (p2->languages) {
+								g_ptr_array_unref(p2->languages);
+							}
+
+							p2->language = sel->language;
+							p2->languages = g_ptr_array_ref(sel->languages);
+						}
+						else {
+							if (p1->languages) {
+								g_ptr_array_unref(p1->languages);
+							}
+
+							p1->language = sel->language;
+							p1->languages = g_ptr_array_ref(sel->languages);
+						}
+					}
+
+					tw = p1->normalized_hashes->len + p2->normalized_hashes->len;
+
+					if (tw > 0) {
+						dw = rspamd_words_levenshtein_distance(task,
+															   p1->normalized_hashes,
+															   p2->normalized_hashes);
+						diff = dw / (gdouble) tw;
+
+						msg_debug_task(
+							"different words: %d, total words: %d, "
+							"got diff between parts of %.2f",
+							dw, tw,
+							diff);
+
+						pdiff = rspamd_mempool_alloc(task->task_pool,
+													 sizeof(gdouble));
+						*pdiff = diff;
+						rspamd_mempool_set_variable(task->task_pool,
+													"parts_distance",
+													pdiff,
+													NULL);
+						ptw = rspamd_mempool_alloc(task->task_pool,
+												   sizeof(gint));
+						*ptw = tw;
+						rspamd_mempool_set_variable(task->task_pool,
+													"total_words",
+													ptw,
+													NULL);
+					}
+				}
+			}
+		}
+		else {
+			debug_task(
+				"message contains two parts but they are in different multi-parts");
+		}
+	}
+
+	if (total_words > 0) {
+		var = rspamd_mempool_get_variable(task->task_pool,
+										  RSPAMD_MEMPOOL_AVG_WORDS_LEN);
+
+		if (var) {
+			*var /= (double) total_words;
+		}
+
+		var = rspamd_mempool_get_variable(task->task_pool,
+										  RSPAMD_MEMPOOL_SHORT_WORDS_CNT);
+
+		if (var) {
+			*var /= (double) total_words;
+		}
+	}
+
+	rspamd_images_link(task);
+	rspamd_tokenize_meta_words(task);
+}
+
+
+struct rspamd_message *
+rspamd_message_ref(struct rspamd_message *msg)
+{
+	REF_RETAIN(msg);
+
+	return msg;
+}
+
+void rspamd_message_unref(struct rspamd_message *msg)
+{
+	if (msg) {
+		REF_RELEASE(msg);
+	}
+}
+
+void rspamd_message_update_digest(struct rspamd_message *msg,
+								  const void *input, gsize len)
+{
+	guint64 n[2];
+	/* Sanity */
+	G_STATIC_ASSERT(sizeof(n) == sizeof(msg->digest));
+
+	memcpy(n, msg->digest, sizeof(msg->digest));
+	n[0] = t1ha2_atonce128(&n[1], input, len, n[0]);
+	memcpy(msg->digest, n, sizeof(msg->digest));
+}
diff --git a/src/libmime/message.h b/src/libmime/message.h
new file mode 100644
index 0000000..52dedab
--- /dev/null
+++ b/src/libmime/message.h
@@ -0,0 +1,239 @@
+/**
+ * @file message.h
+ * Message processing functions and structures
+ */
+
+#ifndef RSPAMD_MESSAGE_H
+#define RSPAMD_MESSAGE_H
+
+#include "config.h"
+
+#include "libmime/email_addr.h"
+#include "libutil/addr.h"
+#include "libcryptobox/cryptobox.h"
+#include "libmime/mime_headers.h"
+#include "libmime/content_type.h"
+#include "libserver/url.h"
+#include "libutil/ref.h"
+#include "libutil/str_util.h"
+
+#include <unicode/uchar.h>
+#include <unicode/utext.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct controller_session;
+struct rspamd_image;
+struct rspamd_archive;
+
+enum rspamd_mime_part_flags {
+	RSPAMD_MIME_PART_ATTACHEMENT = (1u << 1u),
+	RSPAMD_MIME_PART_BAD_CTE = (1u << 4u),
+	RSPAMD_MIME_PART_MISSING_CTE = (1u << 5u),
+	RSPAMD_MIME_PART_NO_TEXT_EXTRACTION = (1u << 6u),
+};
+
+enum rspamd_mime_part_type {
+	RSPAMD_MIME_PART_UNDEFINED = 0,
+	RSPAMD_MIME_PART_MULTIPART,
+	RSPAMD_MIME_PART_MESSAGE,
+	RSPAMD_MIME_PART_TEXT,
+	RSPAMD_MIME_PART_ARCHIVE,
+	RSPAMD_MIME_PART_IMAGE,
+	RSPAMD_MIME_PART_CUSTOM_LUA
+};
+
+#define IS_PART_MULTIPART(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_MULTIPART))
+#define IS_PART_TEXT(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_TEXT))
+#define IS_PART_MESSAGE(part) ((part) && ((part)->part_type == RSPAMD_MIME_PART_MESSAGE))
+
+enum rspamd_cte {
+	RSPAMD_CTE_UNKNOWN = 0,
+	RSPAMD_CTE_7BIT = 1,
+	RSPAMD_CTE_8BIT = 2,
+	RSPAMD_CTE_QP = 3,
+	RSPAMD_CTE_B64 = 4,
+	RSPAMD_CTE_UUE = 5,
+};
+
+struct rspamd_mime_text_part;
+
+struct rspamd_mime_multipart {
+	GPtrArray *children;
+	rspamd_ftok_t boundary;
+};
+
+enum rspamd_lua_specific_type {
+	RSPAMD_LUA_PART_TEXT,
+	RSPAMD_LUA_PART_STRING,
+	RSPAMD_LUA_PART_TABLE,
+	RSPAMD_LUA_PART_FUNCTION,
+	RSPAMD_LUA_PART_UNKNOWN,
+};
+
+struct rspamd_lua_specific_part {
+	gint cbref;
+	enum rspamd_lua_specific_type type;
+};
+
+struct rspamd_mime_part {
+	struct rspamd_content_type *ct;
+	struct rspamd_content_type *detected_ct;
+	gchar *detected_type;
+	gchar *detected_ext;
+	struct rspamd_content_disposition *cd;
+	rspamd_ftok_t raw_data;
+	rspamd_ftok_t parsed_data;
+	struct rspamd_mime_part *parent_part;
+
+	struct rspamd_mime_header *headers_order;
+	struct rspamd_mime_headers_table *raw_headers;
+	GPtrArray *urls;
+
+	gchar *raw_headers_str;
+	gsize raw_headers_len;
+
+	enum rspamd_cte cte;
+	guint flags;
+	enum rspamd_mime_part_type part_type;
+	guint part_number;
+
+	union {
+		struct rspamd_mime_multipart *mp;
+		struct rspamd_mime_text_part *txt;
+		struct rspamd_image *img;
+		struct rspamd_archive *arch;
+		struct rspamd_lua_specific_part lua_specific;
+	} specific;
+
+	guchar digest[rspamd_cryptobox_HASHBYTES];
+};
+
+#define RSPAMD_MIME_TEXT_PART_FLAG_UTF (1 << 0)
+#define RSPAMD_MIME_TEXT_PART_FLAG_EMPTY (1 << 1)
+#define RSPAMD_MIME_TEXT_PART_FLAG_HTML (1 << 2)
+#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW (1 << 3)
+#define RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED (1 << 4)
+#define RSPAMD_MIME_TEXT_PART_ATTACHMENT (1 << 5)
+
+#define IS_TEXT_PART_EMPTY(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_EMPTY)
+#define IS_TEXT_PART_UTF(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+#define IS_TEXT_PART_HTML(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_FLAG_HTML)
+#define IS_TEXT_PART_ATTACHMENT(part) ((part)->flags & RSPAMD_MIME_TEXT_PART_ATTACHMENT)
+
+
+struct rspamd_mime_text_part {
+	const gchar *language;
+	GPtrArray *languages;
+	const gchar *real_charset;
+
+	/* Raw data in native encoding */
+	rspamd_ftok_t raw;
+	rspamd_ftok_t parsed; /* decoded from mime encodings */
+
+	/* UTF8 content */
+	rspamd_ftok_t utf_content;        /* utf8 encoded processed content */
+	GByteArray *utf_raw_content;      /* utf raw content */
+	GByteArray *utf_stripped_content; /* utf content with no newlines */
+	GArray *normalized_hashes;        /* Array of guint64 */
+	GArray *utf_words;                /* Array of rspamd_stat_token_t */
+	UText utf_stripped_text;          /* Used by libicu to represent the utf8 content */
+
+	GPtrArray *newlines; /**< positions of newlines in text, relative to content*/
+	void *html;
+	GList *exceptions; /**< list of offsets of urls						*/
+	struct rspamd_mime_part *mime_part;
+
+	guint flags;
+	guint nlines;
+	guint spaces;
+	guint nwords;
+	guint non_ascii_chars;
+	guint ascii_chars;
+	guint double_spaces;
+	guint non_spaces;
+	guint empty_lines;
+	guint capital_letters;
+	guint numeric_characters;
+	guint unicode_scripts;
+};
+
+struct rspamd_message_raw_headers_content {
+	const gchar *begin;
+	gsize len;
+	const gchar *body_start;
+};
+
+struct rspamd_message {
+	const gchar *message_id;
+	gchar *subject;
+
+	GPtrArray *parts;      /**< list of parsed parts							*/
+	GPtrArray *text_parts; /**< list of text parts								*/
+	struct rspamd_message_raw_headers_content raw_headers_content;
+	void *received_headers; /**< list of received headers						*/
+	khash_t(rspamd_url_hash) * urls;
+	struct rspamd_mime_headers_table *raw_headers; /**< list of raw headers						*/
+	struct rspamd_mime_header *headers_order;      /**< order of raw headers							*/
+	struct rspamd_task *task;
+	GPtrArray *rcpt_mime;
+	GPtrArray *from_mime;
+	guchar digest[16];
+	enum rspamd_newlines_type nlines_type; /**< type of newlines (detected on most of headers 	*/
+	ref_entry_t ref;
+};
+
+#define MESSAGE_FIELD(task, field) ((task)->message->field)
+#define MESSAGE_FIELD_CHECK(task, field) ((task)->message ? (task)->message->field : (__typeof__((task)->message->field)) NULL)
+
+/**
+ * Parse and pre-process mime message
+ * @param task worker_task object
+ * @return
+ */
+gboolean rspamd_message_parse(struct rspamd_task *task);
+
+/**
+ * Process content in task (e.g. HTML parsing)
+ * @param task
+ */
+void rspamd_message_process(struct rspamd_task *task);
+
+
+/**
+ * Converts string to cte
+ * @param str
+ * @return
+ */
+enum rspamd_cte rspamd_cte_from_string(const gchar *str);
+
+/**
+ * Converts cte to string
+ * @param ct
+ * @return
+ */
+const gchar *rspamd_cte_to_string(enum rspamd_cte ct);
+
+struct rspamd_message *rspamd_message_new(struct rspamd_task *task);
+
+struct rspamd_message *rspamd_message_ref(struct rspamd_message *msg);
+
+void rspamd_message_unref(struct rspamd_message *msg);
+
+/**
+ * Updates digest of the message if modified
+ * @param msg
+ * @param input
+ * @param len
+ */
+void rspamd_message_update_digest(struct rspamd_message *msg,
+								  const void *input, gsize len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libmime/mime_encoding.c b/src/libmime/mime_encoding.c
new file mode 100644
index 0000000..48a97a4
--- /dev/null
+++ b/src/libmime/mime_encoding.c
@@ -0,0 +1,864 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "libutil/regexp.h"
+#include "libutil/hash.h"
+#include "libserver/cfg_file.h"
+#include "libserver/task.h"
+#include "mime_encoding.h"
+#include "message.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include "contrib/google-ced/ced_c.h"
+#include <unicode/ucnv.h>
+#if U_ICU_VERSION_MAJOR_NUM >= 44
+#include <unicode/unorm2.h>
+#endif
+#include <math.h>
+
+#define UTF8_CHARSET "UTF-8"
+
+#define RSPAMD_CHARSET_FLAG_UTF (1 << 0)
+#define RSPAMD_CHARSET_FLAG_ASCII (1 << 1)
+
+#define RSPAMD_CHARSET_CACHE_SIZE 32
+#define RSPAMD_CHARSET_MAX_CONTENT 512
+
+#define SET_PART_RAW(part) ((part)->flags &= ~RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+#define SET_PART_UTF(part) ((part)->flags |= RSPAMD_MIME_TEXT_PART_FLAG_UTF)
+
+static rspamd_regexp_t *utf_compatible_re = NULL;
+
+struct rspamd_charset_substitution {
+	const gchar *input;
+	const gchar *canon;
+	gint flags;
+};
+
+#include "mime_encoding_list.h"
+
+static GHashTable *sub_hash = NULL;
+
+static const UChar iso_8859_16_map[] = {
+	0x0080, 0x0081, 0x0082, 0x0083, 0x0084, 0x0085, 0x0086, 0x0087,
+	0x0088, 0x0089, 0x008A, 0x008B, 0x008C, 0x008D, 0x008E, 0x008F,
+	0x0090, 0x0091, 0x0092, 0x0093, 0x0094, 0x0095, 0x0096, 0x0097,
+	0x0098, 0x0099, 0x009A, 0x009B, 0x009C, 0x009D, 0x009E, 0x009F,
+	0x00A0, 0x0104, 0x0105, 0x0141, 0x20AC, 0x201E, 0x0160, 0x00A7,
+	0x0161, 0x00A9, 0x0218, 0x00AB, 0x0179, 0x00AD, 0x017A, 0x017B,
+	0x00B0, 0x00B1, 0x010C, 0x0142, 0x017D, 0x201D, 0x00B6, 0x00B7,
+	0x017E, 0x010D, 0x0219, 0x00BB, 0x0152, 0x0153, 0x0178, 0x017C,
+	0x00C0, 0x00C1, 0x00C2, 0x0102, 0x00C4, 0x0106, 0x00C6, 0x00C7,
+	0x00C8, 0x00C9, 0x00CA, 0x00CB, 0x00CC, 0x00CD, 0x00CE, 0x00CF,
+	0x0110, 0x0143, 0x00D2, 0x00D3, 0x00D4, 0x0150, 0x00D6, 0x015A,
+	0x0170, 0x00D9, 0x00DA, 0x00DB, 0x00DC, 0x0118, 0x021A, 0x00DF,
+	0x00E0, 0x00E1, 0x00E2, 0x0103, 0x00E4, 0x0107, 0x00E6, 0x00E7,
+	0x00E8, 0x00E9, 0x00EA, 0x00EB, 0x00EC, 0x00ED, 0x00EE, 0x00EF,
+	0x0111, 0x0144, 0x00F2, 0x00F3, 0x00F4, 0x0151, 0x00F6, 0x015B,
+	0x0171, 0x00F9, 0x00FA, 0x00FB, 0x00FC, 0x0119, 0x021B, 0x00FF};
+
+struct rspamd_charset_converter {
+	gchar *canon_name;
+	union {
+		UConverter *conv;
+		const UChar *cnv_table;
+	} d;
+	gboolean is_internal;
+};
+
+static GQuark
+rspamd_charset_conv_error_quark(void)
+{
+	return g_quark_from_static_string("charset conversion error");
+}
+
+static void
+rspamd_converter_dtor(gpointer p)
+{
+	struct rspamd_charset_converter *c = (struct rspamd_charset_converter *) p;
+
+	if (!c->is_internal) {
+		ucnv_close(c->d.conv);
+	}
+
+	g_free(c->canon_name);
+	g_free(c);
+}
+
+int32_t
+rspamd_converter_to_uchars(struct rspamd_charset_converter *cnv,
+						   UChar *dest,
+						   int32_t destCapacity,
+						   const char *src,
+						   int32_t srcLength,
+						   UErrorCode *pErrorCode)
+{
+	if (!cnv->is_internal) {
+		return ucnv_toUChars(cnv->d.conv,
+							 dest, destCapacity,
+							 src, srcLength,
+							 pErrorCode);
+	}
+	else {
+		UChar *d = dest, *dend = dest + destCapacity;
+		const guchar *p = src, *end = src + srcLength;
+
+		while (p < end && d < dend) {
+			if (*p <= 127) {
+				*d++ = (UChar) *p;
+			}
+			else {
+				*d++ = cnv->d.cnv_table[*p - 128];
+			}
+
+			p++;
+		}
+
+		return d - dest;
+	}
+}
+
+
+struct rspamd_charset_converter *
+rspamd_mime_get_converter_cached(const gchar *enc,
+								 rspamd_mempool_t *pool,
+								 gboolean is_canon,
+								 UErrorCode *err)
+{
+	const gchar *canon_name;
+	static rspamd_lru_hash_t *cache;
+	struct rspamd_charset_converter *conv;
+
+	if (cache == NULL) {
+		cache = rspamd_lru_hash_new_full(RSPAMD_CHARSET_CACHE_SIZE, NULL,
+										 rspamd_converter_dtor, rspamd_str_hash,
+										 rspamd_str_equal);
+	}
+
+	if (enc == NULL) {
+		return NULL;
+	}
+
+	if (!is_canon) {
+		rspamd_ftok_t cset_tok;
+
+		RSPAMD_FTOK_FROM_STR(&cset_tok, enc);
+		canon_name = rspamd_mime_detect_charset(&cset_tok, pool);
+	}
+	else {
+		canon_name = enc;
+	}
+
+	if (canon_name == NULL) {
+		return NULL;
+	}
+
+	conv = rspamd_lru_hash_lookup(cache, (gpointer) canon_name, 0);
+
+	if (conv == NULL) {
+		if (!(strcmp(canon_name, "ISO-8859-16") == 0 ||
+			  strcmp(canon_name, "latin10") == 0 ||
+			  strcmp(canon_name, "iso-ir-226") == 0)) {
+			conv = g_malloc0(sizeof(*conv));
+			conv->d.conv = ucnv_open(canon_name, err);
+			conv->canon_name = g_strdup(canon_name);
+
+			if (conv->d.conv != NULL) {
+				ucnv_setToUCallBack(conv->d.conv,
+									UCNV_TO_U_CALLBACK_SUBSTITUTE,
+									NULL,
+									NULL,
+									NULL,
+									err);
+				rspamd_lru_hash_insert(cache, conv->canon_name, conv, 0, 0);
+			}
+			else {
+				g_free(conv);
+				conv = NULL;
+			}
+		}
+		else {
+			/* ISO-8859-16 */
+			conv = g_malloc0(sizeof(*conv));
+			conv->is_internal = TRUE;
+			conv->d.cnv_table = iso_8859_16_map;
+			conv->canon_name = g_strdup(canon_name);
+
+			rspamd_lru_hash_insert(cache, conv->canon_name, conv, 0, 0);
+		}
+	}
+
+	return conv;
+}
+
+static void
+rspamd_mime_encoding_substitute_init(void)
+{
+	guint i;
+
+	sub_hash = g_hash_table_new(rspamd_strcase_hash, rspamd_strcase_equal);
+
+	for (i = 0; i < G_N_ELEMENTS(sub); i++) {
+		g_hash_table_insert(sub_hash, (void *) sub[i].input, (void *) &sub[i]);
+	}
+}
+
+static void
+rspamd_charset_normalize(gchar *in)
+{
+	/*
+	 * This is a simple routine to validate input charset
+	 * we just check that charset starts with alphanumeric and ends
+	 * with alphanumeric
+	 */
+	gchar *begin, *end;
+	gboolean changed = FALSE;
+
+	begin = in;
+
+	while (*begin && !g_ascii_isalnum(*begin)) {
+		begin++;
+		changed = TRUE;
+	}
+
+	end = begin + strlen(begin) - 1;
+
+	while (end > begin && !g_ascii_isalnum(*end)) {
+		end--;
+		changed = TRUE;
+	}
+
+	if (changed) {
+		memmove(in, begin, end - begin + 2);
+		*(end + 1) = '\0';
+	}
+}
+
+const gchar *
+rspamd_mime_detect_charset(const rspamd_ftok_t *in, rspamd_mempool_t *pool)
+{
+	gchar *ret = NULL, *h, *t;
+	struct rspamd_charset_substitution *s;
+	const gchar *cset;
+	rspamd_ftok_t utf8_tok;
+	UErrorCode uc_err = U_ZERO_ERROR;
+
+	if (sub_hash == NULL) {
+		rspamd_mime_encoding_substitute_init();
+	}
+
+	/* Fast path */
+	RSPAMD_FTOK_ASSIGN(&utf8_tok, "utf-8");
+
+	if (rspamd_ftok_casecmp(in, &utf8_tok) == 0) {
+		return UTF8_CHARSET;
+	}
+
+	RSPAMD_FTOK_ASSIGN(&utf8_tok, "utf8");
+
+	if (rspamd_ftok_casecmp(in, &utf8_tok) == 0) {
+		return UTF8_CHARSET;
+	}
+
+	ret = rspamd_mempool_ftokdup(pool, in);
+	rspamd_charset_normalize(ret);
+
+	if ((in->len > 3 && rspamd_lc_cmp(in->begin, "cp-", 3) == 0) ||
+		(in->len > 4 && (rspamd_lc_cmp(in->begin, "ibm-", 4) == 0))) {
+		/* Try to remove '-' chars from encoding: e.g. CP-100 to CP100 */
+		h = ret;
+		t = ret;
+
+		while (*h != '\0') {
+			if (*h != '-') {
+				*t++ = *h;
+			}
+
+			h++;
+		}
+
+		*t = '\0';
+	}
+
+	s = g_hash_table_lookup(sub_hash, ret);
+
+	if (s) {
+		ret = (char *) s->canon;
+	}
+
+	/* Try different aliases */
+	cset = ucnv_getCanonicalName(ret, "MIME", &uc_err);
+
+	if (cset == NULL) {
+		uc_err = U_ZERO_ERROR;
+		cset = ucnv_getCanonicalName(ret, "IANA", &uc_err);
+	}
+
+	if (cset == NULL) {
+		uc_err = U_ZERO_ERROR;
+		cset = ucnv_getCanonicalName(ret, "", &uc_err);
+	}
+
+	if (cset == NULL) {
+		uc_err = U_ZERO_ERROR;
+		cset = ucnv_getAlias(ret, 0, &uc_err);
+	}
+
+	return cset;
+}
+
+gchar *
+rspamd_mime_text_to_utf8(rspamd_mempool_t *pool,
+						 gchar *input, gsize len, const gchar *in_enc,
+						 gsize *olen, GError **err)
+{
+	gchar *d;
+	gint32 r, clen, dlen;
+	UChar *tmp_buf;
+
+	UErrorCode uc_err = U_ZERO_ERROR;
+	UConverter *utf8_converter;
+	struct rspamd_charset_converter *conv;
+	rspamd_ftok_t cset_tok;
+
+	/* Check if already utf8 */
+	RSPAMD_FTOK_FROM_STR(&cset_tok, in_enc);
+
+	if (rspamd_mime_charset_utf_check(&cset_tok, input, len,
+									  FALSE)) {
+		d = rspamd_mempool_alloc(pool, len);
+		memcpy(d, input, len);
+		if (olen) {
+			*olen = len;
+		}
+
+		return d;
+	}
+
+	conv = rspamd_mime_get_converter_cached(in_enc, pool, TRUE, &uc_err);
+	utf8_converter = rspamd_get_utf8_converter();
+
+	if (conv == NULL) {
+		g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+					"cannot open converter for %s: %s",
+					in_enc, u_errorName(uc_err));
+
+		return NULL;
+	}
+
+	tmp_buf = g_new(UChar, len + 1);
+	uc_err = U_ZERO_ERROR;
+	r = rspamd_converter_to_uchars(conv, tmp_buf, len + 1, input, len, &uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+		g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+					"cannot convert data to unicode from %s: %s",
+					in_enc, u_errorName(uc_err));
+		g_free(tmp_buf);
+
+		return NULL;
+	}
+
+	/* Now, convert to utf8 */
+	clen = ucnv_getMaxCharSize(utf8_converter);
+	dlen = UCNV_GET_MAX_BYTES_FOR_STRING(r, clen);
+	d = rspamd_mempool_alloc(pool, dlen);
+	r = ucnv_fromUChars(utf8_converter, d, dlen, tmp_buf, r, &uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+		g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+					"cannot convert data from unicode from %s: %s",
+					in_enc, u_errorName(uc_err));
+		g_free(tmp_buf);
+
+		return NULL;
+	}
+
+	msg_debug_pool("converted from %s to UTF-8 inlen: %z, outlen: %d",
+				   in_enc, len, r);
+	g_free(tmp_buf);
+
+	if (olen) {
+		*olen = r;
+	}
+
+	return d;
+}
+
+static gboolean
+rspamd_mime_text_part_utf8_convert(struct rspamd_task *task,
+								   struct rspamd_mime_text_part *text_part,
+								   GByteArray *input,
+								   const gchar *charset,
+								   GError **err)
+{
+	gchar *d;
+	gint32 r, clen, dlen, uc_len;
+	UChar *tmp_buf;
+	UErrorCode uc_err = U_ZERO_ERROR;
+	UConverter *utf8_converter;
+	struct rspamd_charset_converter *conv;
+
+	conv = rspamd_mime_get_converter_cached(charset, task->task_pool,
+											TRUE, &uc_err);
+	utf8_converter = rspamd_get_utf8_converter();
+
+	if (conv == NULL) {
+		g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+					"cannot open converter for %s: %s",
+					charset, u_errorName(uc_err));
+
+		return FALSE;
+	}
+
+	tmp_buf = g_new(UChar, input->len + 1);
+	uc_err = U_ZERO_ERROR;
+	uc_len = rspamd_converter_to_uchars(conv,
+										tmp_buf,
+										input->len + 1,
+										input->data,
+										input->len,
+										&uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+		g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+					"cannot convert data to unicode from %s: %s",
+					charset, u_errorName(uc_err));
+		g_free(tmp_buf);
+
+		return FALSE;
+	}
+
+	/* Now, convert to utf8 */
+	clen = ucnv_getMaxCharSize(utf8_converter);
+	dlen = UCNV_GET_MAX_BYTES_FOR_STRING(uc_len, clen);
+	d = rspamd_mempool_alloc(task->task_pool, dlen);
+	r = ucnv_fromUChars(utf8_converter, d, dlen,
+						tmp_buf, uc_len, &uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+		g_set_error(err, rspamd_charset_conv_error_quark(), EINVAL,
+					"cannot convert data from unicode from %s: %s",
+					charset, u_errorName(uc_err));
+		g_free(tmp_buf);
+
+		return FALSE;
+	}
+
+	if (text_part->mime_part && text_part->mime_part->ct) {
+		msg_info_task("converted text part from %s ('%T' announced) to UTF-8 inlen: %d, outlen: %d (%d UTF16 chars)",
+					  charset, &text_part->mime_part->ct->charset, input->len, r, uc_len);
+	}
+	else {
+		msg_info_task("converted text part from %s (no charset announced) to UTF-8 inlen: %d, "
+					  "outlen: %d (%d UTF16 chars)",
+					  charset, input->len, r, uc_len);
+	}
+
+	text_part->utf_raw_content = rspamd_mempool_alloc(task->task_pool,
+													  sizeof(*text_part->utf_raw_content) + sizeof(gpointer) * 4);
+	text_part->utf_raw_content->data = d;
+	text_part->utf_raw_content->len = r;
+	g_free(tmp_buf);
+
+	return TRUE;
+}
+
+gboolean
+rspamd_mime_to_utf8_byte_array(GByteArray *in,
+							   GByteArray *out,
+							   rspamd_mempool_t *pool,
+							   const gchar *enc)
+{
+	gint32 r, clen, dlen;
+	UChar *tmp_buf;
+	UErrorCode uc_err = U_ZERO_ERROR;
+	UConverter *utf8_converter;
+	struct rspamd_charset_converter *conv;
+	rspamd_ftok_t charset_tok;
+
+	if (in == NULL || in->len == 0) {
+		return FALSE;
+	}
+
+	if (enc == NULL) {
+		/* Assume utf ? */
+		if (rspamd_fast_utf8_validate(in->data, in->len) == 0) {
+			g_byte_array_set_size(out, in->len);
+			memcpy(out->data, in->data, out->len);
+
+			return TRUE;
+		}
+		else {
+			/* Bad stuff, keep out */
+			return FALSE;
+		}
+	}
+
+	RSPAMD_FTOK_FROM_STR(&charset_tok, enc);
+
+	if (rspamd_mime_charset_utf_check(&charset_tok, (gchar *) in->data, in->len,
+									  FALSE)) {
+		g_byte_array_set_size(out, in->len);
+		memcpy(out->data, in->data, out->len);
+
+		return TRUE;
+	}
+
+	utf8_converter = rspamd_get_utf8_converter();
+	conv = rspamd_mime_get_converter_cached(enc, pool, TRUE, &uc_err);
+
+	if (conv == NULL) {
+		return FALSE;
+	}
+
+	tmp_buf = g_new(UChar, in->len + 1);
+	uc_err = U_ZERO_ERROR;
+	r = rspamd_converter_to_uchars(conv,
+								   tmp_buf, in->len + 1,
+								   in->data, in->len, &uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+		g_free(tmp_buf);
+
+		return FALSE;
+	}
+
+	/* Now, convert to utf8 */
+	clen = ucnv_getMaxCharSize(utf8_converter);
+	dlen = UCNV_GET_MAX_BYTES_FOR_STRING(r, clen);
+	g_byte_array_set_size(out, dlen);
+	r = ucnv_fromUChars(utf8_converter, out->data, dlen, tmp_buf, r, &uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+		g_free(tmp_buf);
+
+		return FALSE;
+	}
+
+	g_free(tmp_buf);
+	out->len = r;
+
+	return TRUE;
+}
+
+void rspamd_mime_charset_utf_enforce(gchar *in, gsize len)
+{
+	gchar *p, *end;
+	goffset err_offset;
+	UChar32 uc = 0;
+
+	/* Now we validate input and replace bad characters with '?' symbol */
+	p = in;
+	end = in + len;
+
+	while (p < end && len > 0 && (err_offset = rspamd_fast_utf8_validate(p, len)) > 0) {
+		err_offset--; /* As it returns it 1 indexed */
+		gint32 cur_offset = err_offset;
+
+		while (cur_offset < len) {
+			gint32 tmp = cur_offset;
+
+			U8_NEXT(p, cur_offset, len, uc);
+
+			if (uc > 0) {
+				/* Fill string between err_offset and tmp with `?` character */
+				memset(p + err_offset, '?', tmp - err_offset);
+				break;
+			}
+		}
+
+		if (uc < 0) {
+			/* Fill till the end */
+			memset(p + err_offset, '?', len - err_offset);
+			break;
+		}
+
+		p += cur_offset;
+		len = end - p;
+	}
+}
+
+const char *
+rspamd_mime_charset_find_by_content(const gchar *in, gsize inlen,
+									bool check_utf8)
+{
+	int nconsumed;
+	bool is_reliable;
+	const gchar *ced_name;
+
+	if (check_utf8) {
+		if (rspamd_fast_utf8_validate(in, inlen) == 0) {
+			return UTF8_CHARSET;
+		}
+	}
+
+
+	ced_name = ced_encoding_detect(in, inlen, NULL, NULL,
+								   NULL, 0, CED_EMAIL_CORPUS,
+								   false, &nconsumed, &is_reliable);
+
+	if (ced_name) {
+
+		return ced_name;
+	}
+
+	return NULL;
+}
+
+static const char *
+rspamd_mime_charset_find_by_content_maybe_split(const gchar *in, gsize inlen)
+{
+	if (inlen < RSPAMD_CHARSET_MAX_CONTENT * 3) {
+		return rspamd_mime_charset_find_by_content(in, inlen, false);
+	}
+	else {
+		const gchar *c1, *c2, *c3;
+
+		c1 = rspamd_mime_charset_find_by_content(in, RSPAMD_CHARSET_MAX_CONTENT, false);
+		c2 = rspamd_mime_charset_find_by_content(in + inlen / 2,
+												 RSPAMD_CHARSET_MAX_CONTENT, false);
+		c3 = rspamd_mime_charset_find_by_content(in + inlen - RSPAMD_CHARSET_MAX_CONTENT,
+												 RSPAMD_CHARSET_MAX_CONTENT, false);
+
+		/* 7bit stuff */
+		if (c1 && strcmp(c1, "US-ASCII") == 0) {
+			c1 = NULL; /* Invalid - we have 8 bit there */
+		}
+		if (c2 && strcmp(c2, "US-ASCII") == 0) {
+			c2 = NULL; /* Invalid - we have 8 bit there */
+		}
+		if (c3 && strcmp(c3, "US-ASCII") == 0) {
+			c3 = NULL; /* Invalid - we have 8 bit there */
+		}
+
+		if (!c1) {
+			c1 = c2 ? c2 : c3;
+		}
+		if (!c2) {
+			c2 = c3 ? c3 : c1;
+		}
+		if (!c3) {
+			c3 = c1 ? c2 : c1;
+		}
+
+		if (c1 && c2 && c3) {
+			/* Quorum */
+			if (c1 == c2) {
+				return c1;
+			}
+			else if (c2 == c3) {
+				return c2;
+			}
+			else if (c1 == c3) {
+				return c3;
+			}
+
+			/* All charsets are distinct. Use the one from the top */
+			return c1;
+		}
+
+		return NULL;
+	}
+}
+
+gboolean
+rspamd_mime_charset_utf_check(rspamd_ftok_t *charset,
+							  gchar *in, gsize len, gboolean content_check)
+{
+	const gchar *real_charset;
+
+	if (utf_compatible_re == NULL) {
+		utf_compatible_re = rspamd_regexp_new(
+			"^(?:utf-?8.*)|(?:us-ascii)|(?:ascii)|(?:ansi.*)|(?:CSASCII)$",
+			"i", NULL);
+	}
+
+	if (charset->len == 0 ||
+		rspamd_regexp_match(utf_compatible_re,
+							charset->begin, charset->len, TRUE)) {
+		/*
+		 * In case of UTF8 charset we still can check the content to find
+		 * corner cases
+		 */
+		if (content_check) {
+			if (rspamd_fast_utf8_validate(in, len) != 0) {
+				real_charset = rspamd_mime_charset_find_by_content_maybe_split(in, len);
+
+				if (real_charset) {
+
+					if (rspamd_regexp_match(utf_compatible_re,
+											real_charset, strlen(real_charset), TRUE)) {
+						RSPAMD_FTOK_ASSIGN(charset, UTF8_CHARSET);
+
+						return TRUE;
+					}
+					else {
+						charset->begin = real_charset;
+						charset->len = strlen(real_charset);
+
+						return FALSE;
+					}
+				}
+
+				rspamd_mime_charset_utf_enforce(in, len);
+			}
+		}
+
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+void rspamd_mime_text_part_maybe_convert(struct rspamd_task *task,
+										 struct rspamd_mime_text_part *text_part)
+{
+	GError *err = NULL;
+	const gchar *charset = NULL;
+	gboolean checked = FALSE, need_charset_heuristic = TRUE, valid_utf8 = FALSE;
+	GByteArray *part_content;
+	rspamd_ftok_t charset_tok;
+	struct rspamd_mime_part *part = text_part->mime_part;
+
+	if (rspamd_str_has_8bit(text_part->raw.begin, text_part->raw.len)) {
+		text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_8BIT_RAW;
+	}
+
+	/* Allocate copy storage */
+	part_content = g_byte_array_sized_new(text_part->parsed.len);
+	memcpy(part_content->data, text_part->parsed.begin, text_part->parsed.len);
+	part_content->len = text_part->parsed.len;
+	rspamd_mempool_notify_alloc(task->task_pool,
+								part_content->len);
+	rspamd_mempool_add_destructor(task->task_pool,
+								  (rspamd_mempool_destruct_t) g_byte_array_unref, part_content);
+
+	if (rspamd_str_has_8bit(text_part->parsed.begin, text_part->parsed.len)) {
+		if (rspamd_fast_utf8_validate(text_part->parsed.begin, text_part->parsed.len) == 0) {
+			/* Valid UTF, likely all good */
+			need_charset_heuristic = FALSE;
+			valid_utf8 = TRUE;
+			checked = TRUE;
+		}
+
+		text_part->flags |= RSPAMD_MIME_TEXT_PART_FLAG_8BIT_ENCODED;
+	}
+	else {
+		/* All 7bit characters, assume it valid utf */
+		need_charset_heuristic = FALSE;
+		valid_utf8 = TRUE;
+		checked = TRUE; /* Already valid utf, no need in further checks */
+	}
+
+	if (part->ct->charset.len == 0) {
+		if (need_charset_heuristic) {
+			charset = rspamd_mime_charset_find_by_content_maybe_split(text_part->parsed.begin,
+																	  text_part->parsed.len);
+
+			if (charset != NULL) {
+				msg_info_task("detected charset %s", charset);
+			}
+
+			checked = TRUE;
+			text_part->real_charset = charset;
+		}
+		else if (valid_utf8) {
+			SET_PART_UTF(text_part);
+			text_part->utf_raw_content = part_content;
+			text_part->real_charset = UTF8_CHARSET;
+
+			return;
+		}
+	}
+	else {
+		charset = rspamd_mime_detect_charset(&part->ct->charset,
+											 task->task_pool);
+
+		if (charset == NULL) {
+			/* We don't know the real charset but can try heuristic */
+			if (need_charset_heuristic) {
+				charset = rspamd_mime_charset_find_by_content_maybe_split(part_content->data,
+																		  part_content->len);
+				msg_info_task("detected charset: %s", charset);
+				checked = TRUE;
+				text_part->real_charset = charset;
+			}
+			else if (valid_utf8) {
+				/* We already know that the input is valid utf, so skip heuristic */
+				text_part->real_charset = UTF8_CHARSET;
+			}
+		}
+		else {
+			text_part->real_charset = charset;
+
+			if (strcmp(charset, UTF8_CHARSET) != 0) {
+				/*
+				 * We have detected some charset, but we don't know which one,
+				 * so we need to reset valid utf8 flag and enforce it later
+				 */
+				valid_utf8 = FALSE;
+			}
+		}
+	}
+
+	if (text_part->real_charset == NULL) {
+		msg_info_task("<%s>: has invalid charset; original charset: %T; Content-Type: \"%s\"",
+					  MESSAGE_FIELD_CHECK(task, message_id), &part->ct->charset,
+					  part->ct->cpy);
+		SET_PART_RAW(text_part);
+		text_part->utf_raw_content = part_content;
+
+		return;
+	}
+
+	RSPAMD_FTOK_FROM_STR(&charset_tok, charset);
+
+	if (!valid_utf8) {
+		if (rspamd_mime_charset_utf_check(&charset_tok, part_content->data,
+										  part_content->len, !checked)) {
+			SET_PART_UTF(text_part);
+			text_part->utf_raw_content = part_content;
+			text_part->real_charset = UTF8_CHARSET;
+
+			return;
+		}
+		else {
+			charset = charset_tok.begin;
+
+			if (!rspamd_mime_text_part_utf8_convert(task, text_part,
+													part_content, charset, &err)) {
+				msg_warn_task("<%s>: cannot convert from %s to utf8: %s",
+							  MESSAGE_FIELD(task, message_id),
+							  charset,
+							  err ? err->message : "unknown problem");
+				SET_PART_RAW(text_part);
+				g_error_free(err);
+
+				text_part->utf_raw_content = part_content;
+				return;
+			}
+
+			SET_PART_UTF(text_part);
+			text_part->real_charset = charset;
+		}
+	}
+	else {
+		SET_PART_UTF(text_part);
+		text_part->utf_raw_content = part_content;
+	}
+}
diff --git a/src/libmime/mime_encoding.h b/src/libmime/mime_encoding.h
new file mode 100644
index 0000000..ff81292
--- /dev/null
+++ b/src/libmime/mime_encoding.h
@@ -0,0 +1,148 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_MIME_ENCODING_H_
+#define SRC_LIBMIME_MIME_ENCODING_H_
+
+#include "config.h"
+#include "mem_pool.h"
+#include "fstring.h"
+#include <unicode/uchar.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_mime_part;
+struct rspamd_mime_text_part;
+struct rspamd_charset_converter;
+
+/**
+ * Convert charset alias to a canonic charset name
+ * @param pool pool to store temporary data
+ * @param in
+ * @return
+ */
+const gchar *rspamd_mime_detect_charset(const rspamd_ftok_t *in,
+										rspamd_mempool_t *pool);
+
+/**
+ * Convert text chunk to utf-8. Input encoding is substituted using
+ * `rspamd_mime_detect_charset`.
+ * If input encoding is already utf, this function returns input pointer.
+ * Memory is allocated from pool if a conversion is needed
+ * @param pool
+ * @param input
+ * @param len
+ * @param in_enc canon charset
+ * @param olen
+ * @param err
+ * @return
+ */
+gchar *rspamd_mime_text_to_utf8(rspamd_mempool_t *pool,
+								gchar *input, gsize len, const gchar *in_enc,
+								gsize *olen, GError **err);
+
+/**
+ * Converts data from `in` to `out`,
+ * returns `FALSE` if `enc` is not a valid iconv charset
+ *
+ * This function, in fact, copies `in` from `out` replacing out content in
+ * total.
+ * @param in
+ * @param out
+ * @param enc validated canonical charset name. If NULL, then utf8 check is done only
+ * @return
+ */
+gboolean rspamd_mime_to_utf8_byte_array(GByteArray *in,
+										GByteArray *out,
+										rspamd_mempool_t *pool,
+										const gchar *enc);
+
+/**
+ * Maybe convert part to utf-8
+ * @param task
+ * @param text_part
+ * @return
+ */
+void rspamd_mime_text_part_maybe_convert(struct rspamd_task *task,
+										 struct rspamd_mime_text_part *text_part);
+
+/**
+ * Checks utf8 charset and normalize/validate utf8 string
+ * @param charset
+ * @param in
+ * @param len
+ * @return
+ */
+gboolean rspamd_mime_charset_utf_check(rspamd_ftok_t *charset,
+									   gchar *in, gsize len,
+									   gboolean content_check);
+
+/**
+ * Ensure that all characters in string are valid utf8 chars or replace them
+ * with '?'
+ * @param in
+ * @param len
+ */
+void rspamd_mime_charset_utf_enforce(gchar *in, gsize len);
+
+/**
+  * Gets cached converter
+  * @param enc input encoding
+  * @param pool pool to use for temporary normalisation
+  * @param is_canon TRUE if normalisation is needed
+  * @param err output error
+  * @return converter
+  */
+struct rspamd_charset_converter *rspamd_mime_get_converter_cached(
+	const gchar *enc,
+	rspamd_mempool_t *pool,
+	gboolean is_canon,
+	UErrorCode *err);
+
+/**
+ * Performs charset->utf16 conversion
+ * @param cnv
+ * @param dest
+ * @param destCapacity
+ * @param src
+ * @param srcLength
+ * @param pErrorCode
+ * @return
+ */
+gint32
+rspamd_converter_to_uchars(struct rspamd_charset_converter *cnv,
+						   UChar *dest,
+						   gint32 destCapacity,
+						   const char *src,
+						   gint32 srcLength,
+						   UErrorCode *pErrorCode);
+
+/**
+ * Detect charset in text
+ * @param in
+ * @param inlen
+ * @return detected charset name or NULL
+ */
+const char *rspamd_mime_charset_find_by_content(const gchar *in, gsize inlen,
+												bool check_utf8);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_MIME_ENCODING_H_ */
diff --git a/src/libmime/mime_encoding_list.h b/src/libmime/mime_encoding_list.h
new file mode 100644
index 0000000..b5fc5e1
--- /dev/null
+++ b/src/libmime/mime_encoding_list.h
@@ -0,0 +1,1577 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_MIME_ENCODING_LIST_H_
+#define SRC_LIBMIME_MIME_ENCODING_LIST_H_
+
+static const struct rspamd_charset_substitution sub[] = {
+	{
+		.input = "iso-646-us",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "ansi_x3.4-1968",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "iso-ir-6",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "iso_646.irv:1991",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "ascii",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "iso646-us",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "us",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "ibm367",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "cp367",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "csascii",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "ascii7",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "default",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "646",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "iso_646.irv:1983",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "iso969-us",
+		.canon = "ansi_x3.4-1986",
+		.flags = RSPAMD_CHARSET_FLAG_ASCII,
+	},
+	{
+		.input = "tw-big5",
+		.canon = "big5",
+		.flags = 0,
+	},
+	{
+		.input = "csbig5",
+		.canon = "big5",
+		.flags = 0,
+	},
+	{
+		.input = "hkscs-big5",
+		.canon = "big5-hkscs",
+		.flags = 0,
+	},
+	{
+		.input = "big5hk",
+		.canon = "big5-hkscs",
+		.flags = 0,
+	},
+	{
+		.input = "big5-hkscs:unicode",
+		.canon = "big5-hkscs",
+		.flags = 0,
+	},
+	{
+		.input = "extended_unix_code_packed_format_for_japanese",
+		.canon = "euc-jp",
+		.flags = 0,
+	},
+	{
+		.input = "cseucpkdfmtjapanese",
+		.canon = "euc-jp",
+		.flags = 0,
+	},
+	{
+		.input = "x-eucjp",
+		.canon = "euc-jp",
+		.flags = 0,
+	},
+	{
+		.input = "x-euc-jp",
+		.canon = "euc-jp",
+		.flags = 0,
+	},
+	{
+		.input = "unicode-1-1-utf-8",
+		.canon = "utf-8",
+		.flags = RSPAMD_CHARSET_FLAG_UTF,
+	},
+	{
+		.input = "cseuckr",
+		.canon = "euc-kr",
+		.flags = 0,
+	},
+	{
+		.input = "5601",
+		.canon = "euc-kr",
+		.flags = 0,
+	},
+	{
+		.input = "ksc-5601",
+		.canon = "euc-kr",
+		.flags = 0,
+	},
+	{
+		.input = "ksc-5601-1987",
+		.canon = "euc-kr",
+		.flags = 0,
+	},
+	{
+		.input = "ksc-5601_1987",
+		.canon = "euc-kr",
+		.flags = 0,
+	},
+	{
+		.input = "ksc5601",
+		.canon = "euc-kr",
+		.flags = 0,
+	},
+	{
+		.input = "cns11643",
+		.canon = "euc-tw",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-euctw",
+		.canon = "euc-tw",
+		.flags = 0,
+	},
+	{
+		.input = "gb-18030",
+		.canon = "gb18030",
+		.flags = 0,
+	},
+	{
+		.input = "ibm1392",
+		.canon = "gb18030",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-1392",
+		.canon = "gb18030",
+		.flags = 0,
+	},
+	{
+		.input = "gb18030-2000",
+		.canon = "gb18030",
+		.flags = 0,
+	},
+	{
+		.input = "gb-2312",
+		.canon = "gb2312",
+		.flags = 0,
+	},
+	{
+		.input = "csgb2312",
+		.canon = "gb2312",
+		.flags = 0,
+	},
+	{
+		.input = "euc_cn",
+		.canon = "gb2312",
+		.flags = 0,
+	},
+	{
+		.input = "euccn",
+		.canon = "gb2312",
+		.flags = 0,
+	},
+	{
+		.input = "euc-cn",
+		.canon = "gb2312",
+		.flags = 0,
+	},
+	{
+		.input = "gb-k",
+		.canon = "gbk",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-1:1987",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-100",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-1",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "latin1",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "l1",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "ibm819",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "cp819",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatin1",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "819",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "cp819",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-1",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "8859-1",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_1",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_1",
+		.canon = "iso-8859-1",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-2:1987",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-101",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-2",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "latin2",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "l2",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatin2",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "912",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "cp912",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-912",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "ibm912",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-2",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "8859-2",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_2",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_2",
+		.canon = "iso-8859-2",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-3:1988",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-109",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-3",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "latin3",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "l3",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatin3",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "913",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "cp913",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-913",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "ibm913",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-3",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "8859-3",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_3",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_3",
+		.canon = "iso-8859-3",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-4:1988",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-110",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-4",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "latin4",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "l4",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatin4",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "914",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "cp914",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-914",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "ibm914",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-4",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "8859-4",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_4",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_4",
+		.canon = "iso-8859-4",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-5:1988",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-144",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-5",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "cyrillic",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatincyrillic",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "915",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "cp915",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-915",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "ibm915",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-5",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "8859-5",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_5",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_5",
+		.canon = "iso-8859-5",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-6:1987",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-127",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-6",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "ecma-114",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "asmo-708",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "arabic",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatinarabic",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "1089",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "cp1089",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-1089",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "ibm1089",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-6",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "8859-6",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_6",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_6",
+		.canon = "iso-8859-6",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-7:1987",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-126",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-7",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "elot_928",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "ecma-118",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "greek",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "greek8",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatingreek",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "813",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "cp813",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-813",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "ibm813",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-7",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "8859-7",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_7",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_7",
+		.canon = "iso-8859-7",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-8:1988",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-138",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-8",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "hebrew",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatinhebrew",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "916",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "cp916",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-916",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "ibm916",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-8",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "8859-8",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_8",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_8",
+		.canon = "iso-8859-8",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-9:1989",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-148",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-9",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "latin5",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "l5",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatin5",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "920",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "cp920",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-920",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "ibm920",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-9",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "8859-9",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_9",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_9",
+		.canon = "iso-8859-9",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-13",
+		.canon = "iso-8859-13",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-13",
+		.canon = "iso-8859-13",
+		.flags = 0,
+	},
+	{
+		.input = "8859-13",
+		.canon = "iso-8859-13",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859_13",
+		.canon = "iso-8859-13",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859_13",
+		.canon = "iso-8859-13",
+		.flags = 0,
+	},
+	{
+		.input = "iso-ir-199",
+		.canon = "iso-8859-14",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-14:1998",
+		.canon = "iso-8859-14",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-14",
+		.canon = "iso-8859-14",
+		.flags = 0,
+	},
+	{
+		.input = "latin8",
+		.canon = "iso-8859-14",
+		.flags = 0,
+	},
+	{
+		.input = "iso-celtic",
+		.canon = "iso-8859-14",
+		.flags = 0,
+	},
+	{
+		.input = "l8",
+		.canon = "iso-8859-14",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatin9",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "csisolatin0",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "latin9",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "latin0",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "923",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "cp923",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-923",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "ibm923",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "iso8859-15",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-15",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "8859-15",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "iso_8859-15_fdis",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "l9",
+		.canon = "iso-8859-15",
+		.flags = 0,
+	},
+	{
+		.input = "koi-8-r",
+		.canon = "koi8-r",
+		.flags = 0,
+	},
+	{
+		.input = "cskoi8r",
+		.canon = "koi8-r",
+		.flags = 0,
+	},
+	{
+		.input = "koi8",
+		.canon = "koi8-r",
+		.flags = 0,
+	},
+	{
+		.input = "koi-8-u",
+		.canon = "koi8-u",
+		.flags = 0,
+	},
+	{
+		.input = "koi-8-t",
+		.canon = "koi8-t",
+		.flags = 0,
+	},
+	{
+		.input = "shiftjis",
+		.canon = "shift_jis",
+		.flags = 0,
+	},
+	{
+		.input = "ms_kanji",
+		.canon = "shift_jis",
+		.flags = 0,
+	},
+	{
+		.input = "csshiftjis",
+		.canon = "shift_jis",
+		.flags = 0,
+	},
+	{
+		.input = "cp-437",
+		.canon = "ibm437",
+		.flags = 0,
+	},
+	{
+		.input = "cp437",
+		.canon = "ibm437",
+		.flags = 0,
+	},
+	{
+		.input = "437",
+		.canon = "ibm437",
+		.flags = 0,
+	},
+	{
+		.input = "cspc8codepage437437",
+		.canon = "ibm437",
+		.flags = 0,
+	},
+	{
+		.input = "cspc8codepage437",
+		.canon = "ibm437",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-437",
+		.canon = "ibm437",
+		.flags = 0,
+	},
+	{
+		.input = "cp-850",
+		.canon = "ibm850",
+		.flags = 0,
+	},
+	{
+		.input = "cp850",
+		.canon = "ibm850",
+		.flags = 0,
+	},
+	{
+		.input = "850",
+		.canon = "ibm850",
+		.flags = 0,
+	},
+	{
+		.input = "cspc850multilingual850",
+		.canon = "ibm850",
+		.flags = 0,
+	},
+	{
+		.input = "cspc850multilingual",
+		.canon = "ibm850",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-850",
+		.canon = "ibm850",
+		.flags = 0,
+	},
+	{
+		.input = "cp-851",
+		.canon = "ibm851",
+		.flags = 0,
+	},
+	{
+		.input = "cp851",
+		.canon = "ibm851",
+		.flags = 0,
+	},
+	{
+		.input = "851",
+		.canon = "ibm851",
+		.flags = 0,
+	},
+	{
+		.input = "csibm851",
+		.canon = "ibm851",
+		.flags = 0,
+	},
+	{
+		.input = "cp-852",
+		.canon = "ibm852",
+		.flags = 0,
+	},
+	{
+		.input = "cp852",
+		.canon = "ibm852",
+		.flags = 0,
+	},
+	{
+		.input = "852",
+		.canon = "ibm852",
+		.flags = 0,
+	},
+	{
+		.input = "cspcp852",
+		.canon = "ibm852",
+		.flags = 0,
+	},
+	{
+		.input = "852",
+		.canon = "ibm852",
+		.flags = 0,
+	},
+	{
+		.input = "cspcp852",
+		.canon = "ibm852",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-852",
+		.canon = "ibm852",
+		.flags = 0,
+	},
+	{
+		.input = "cp-855",
+		.canon = "ibm855",
+		.flags = 0,
+	},
+	{
+		.input = "cp855",
+		.canon = "ibm855",
+		.flags = 0,
+	},
+	{
+		.input = "855",
+		.canon = "ibm855",
+		.flags = 0,
+	},
+	{
+		.input = "csibm855",
+		.canon = "ibm855",
+		.flags = 0,
+	},
+	{
+		.input = "cspcp855",
+		.canon = "ibm855",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-855",
+		.canon = "ibm855",
+		.flags = 0,
+	},
+	{
+		.input = "cp-857",
+		.canon = "ibm857",
+		.flags = 0,
+	},
+	{
+		.input = "cp857",
+		.canon = "ibm857",
+		.flags = 0,
+	},
+	{
+		.input = "857",
+		.canon = "ibm857",
+		.flags = 0,
+	},
+	{
+		.input = "csibm857",
+		.canon = "ibm857",
+		.flags = 0,
+	},
+	{
+		.input = "857",
+		.canon = "ibm857",
+		.flags = 0,
+	},
+	{
+		.input = "csibm857",
+		.canon = "ibm857",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-857",
+		.canon = "ibm857",
+		.flags = 0,
+	},
+	{
+		.input = "cp-860",
+		.canon = "ibm860",
+		.flags = 0,
+	},
+	{
+		.input = "cp860",
+		.canon = "ibm860",
+		.flags = 0,
+	},
+	{
+		.input = "860",
+		.canon = "ibm860",
+		.flags = 0,
+	},
+	{
+		.input = "csibm860",
+		.canon = "ibm860",
+		.flags = 0,
+	},
+	{
+		.input = "860",
+		.canon = "ibm860",
+		.flags = 0,
+	},
+	{
+		.input = "csibm860",
+		.canon = "ibm860",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-860",
+		.canon = "ibm860",
+		.flags = 0,
+	},
+	{
+		.input = "cp-861",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "cp861",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "861",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "cp-is",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "csibm861",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "861",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "cp-is",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "csibm861",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-861",
+		.canon = "ibm861",
+		.flags = 0,
+	},
+	{
+		.input = "cp-862",
+		.canon = "ibm862",
+		.flags = 0,
+	},
+	{
+		.input = "cp862",
+		.canon = "ibm862",
+		.flags = 0,
+	},
+	{
+		.input = "862",
+		.canon = "ibm862",
+		.flags = 0,
+	},
+	{
+		.input = "cspc862latinhebrew862",
+		.canon = "ibm862",
+		.flags = 0,
+	},
+	{
+		.input = "cspc862latinhebrew",
+		.canon = "ibm862",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-862",
+		.canon = "ibm862",
+		.flags = 0,
+	},
+	{
+		.input = "cp-863",
+		.canon = "ibm863",
+		.flags = 0,
+	},
+	{
+		.input = "cp863",
+		.canon = "ibm863",
+		.flags = 0,
+	},
+	{
+		.input = "863",
+		.canon = "ibm863",
+		.flags = 0,
+	},
+	{
+		.input = "csibm863",
+		.canon = "ibm863",
+		.flags = 0,
+	},
+	{
+		.input = "863",
+		.canon = "ibm863",
+		.flags = 0,
+	},
+	{
+		.input = "csibm863",
+		.canon = "ibm863",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-863",
+		.canon = "ibm863",
+		.flags = 0,
+	},
+	{
+		.input = "cp-864",
+		.canon = "ibm864",
+		.flags = 0,
+	},
+	{
+		.input = "cp864",
+		.canon = "ibm864",
+		.flags = 0,
+	},
+	{
+		.input = "csibm864",
+		.canon = "ibm864",
+		.flags = 0,
+	},
+	{
+		.input = "csibm864",
+		.canon = "ibm864",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-864",
+		.canon = "ibm864",
+		.flags = 0,
+	},
+	{
+		.input = "cp-865",
+		.canon = "ibm865",
+		.flags = 0,
+	},
+	{
+		.input = "cp865",
+		.canon = "ibm865",
+		.flags = 0,
+	},
+	{
+		.input = "865",
+		.canon = "ibm865",
+		.flags = 0,
+	},
+	{
+		.input = "csibm865",
+		.canon = "ibm865",
+		.flags = 0,
+	},
+	{
+		.input = "865",
+		.canon = "ibm865",
+		.flags = 0,
+	},
+	{
+		.input = "csibm865",
+		.canon = "ibm865",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-865",
+		.canon = "ibm865",
+		.flags = 0,
+	},
+	{
+		.input = "cp-866",
+		.canon = "ibm866",
+		.flags = 0,
+	},
+	{
+		.input = "cp866",
+		.canon = "ibm866",
+		.flags = 0,
+	},
+	{
+		.input = "866",
+		.canon = "ibm866",
+		.flags = 0,
+	},
+	{
+		.input = "csibm866",
+		.canon = "ibm866",
+		.flags = 0,
+	},
+	{
+		.input = "866",
+		.canon = "ibm866",
+		.flags = 0,
+	},
+	{
+		.input = "csibm866",
+		.canon = "ibm866",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-866",
+		.canon = "ibm866",
+		.flags = 0,
+	},
+	{
+		.input = "cp-868",
+		.canon = "ibm868",
+		.flags = 0,
+	},
+	{
+		.input = "cp868",
+		.canon = "ibm868",
+		.flags = 0,
+	},
+	{
+		.input = "cp-ar",
+		.canon = "ibm868",
+		.flags = 0,
+	},
+	{
+		.input = "csibm868",
+		.canon = "ibm868",
+		.flags = 0,
+	},
+	{
+		.input = "ibm-868",
+		.canon = "ibm868",
+		.flags = 0,
+	},
+	{
+		.input = "cp-869",
+		.canon = "ibm869",
+		.flags = 0,
+	},
+	{
+		.input = "cp869",
+		.canon = "ibm869",
+		.flags = 0,
+	},
+	{
+		.input = "869",
+		.canon = "ibm869",
+		.flags = 0,
+	},
+	{
+		.input = "cp-gr",
+		.canon = "ibm869",
+		.flags = 0,
+	},
+	{
+		.input = "csibm869",
+		.canon = "ibm869",
+		.flags = 0,
+	},
+	{
+		.input = "cp-891",
+		.canon = "ibm891",
+		.flags = 0,
+	},
+	{
+		.input = "cp891",
+		.canon = "ibm891",
+		.flags = 0,
+	},
+	{
+		.input = "csibm891",
+		.canon = "ibm891",
+		.flags = 0,
+	},
+	{
+		.input = "cp-903",
+		.canon = "ibm903",
+		.flags = 0,
+	},
+	{
+		.input = "cp903",
+		.canon = "ibm903",
+		.flags = 0,
+	},
+	{
+		.input = "csibm903",
+		.canon = "ibm903",
+		.flags = 0,
+	},
+	{
+		.input = "cp-904",
+		.canon = "ibm904",
+		.flags = 0,
+	},
+	{
+		.input = "cp904",
+		.canon = "ibm904",
+		.flags = 0,
+	},
+	{
+		.input = "904",
+		.canon = "ibm904",
+		.flags = 0,
+	},
+	{
+		.input = "csibm904",
+		.canon = "ibm904",
+		.flags = 0,
+	},
+	{
+		.input = "cp-1251",
+		.canon = "cp1251",
+		.flags = 0,
+	},
+	{
+		.input = "windows-1251",
+		.canon = "cp1251",
+		.flags = 0,
+	},
+	{
+		.input = "cp-1255",
+		.canon = "cp1255",
+		.flags = 0,
+	},
+	{
+		.input = "windows-1255",
+		.canon = "cp1255",
+		.flags = 0,
+	},
+	{
+		.input = "tis620.2533",
+		.canon = "tis-620",
+		.flags = 0,
+	},
+};
+
+#endif /* SRC_LIBMIME_MIME_ENCODING_LIST_H_ */
diff --git a/src/libmime/mime_expressions.c b/src/libmime/mime_expressions.c
new file mode 100644
index 0000000..e51539e
--- /dev/null
+++ b/src/libmime/mime_expressions.c
@@ -0,0 +1,2392 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include <contrib/libucl/ucl.h>
+#include "config.h"
+#include "util.h"
+#include "cfg_file.h"
+#include "rspamd.h"
+#include "message.h"
+#include "mime_expressions.h"
+#include "libserver/html/html.h"
+#include "lua/lua_common.h"
+#include "utlist.h"
+
+gboolean rspamd_compare_encoding(struct rspamd_task *task,
+								 GArray *args,
+								 void *unused);
+gboolean rspamd_header_exists(struct rspamd_task *task,
+							  GArray *args,
+							  void *unused);
+gboolean rspamd_parts_distance(struct rspamd_task *task,
+							   GArray *args,
+							   void *unused);
+gboolean rspamd_recipients_distance(struct rspamd_task *task,
+									GArray *args,
+									void *unused);
+gboolean rspamd_has_only_html_part(struct rspamd_task *task,
+								   GArray *args,
+								   void *unused);
+gboolean rspamd_is_recipients_sorted(struct rspamd_task *task,
+									 GArray *args,
+									 void *unused);
+gboolean rspamd_compare_transfer_encoding(struct rspamd_task *task,
+										  GArray *args,
+										  void *unused);
+gboolean rspamd_is_html_balanced(struct rspamd_task *task,
+								 GArray *args,
+								 void *unused);
+gboolean rspamd_has_html_tag(struct rspamd_task *task,
+							 GArray *args,
+							 void *unused);
+gboolean rspamd_has_fake_html(struct rspamd_task *task,
+							  GArray *args,
+							  void *unused);
+static gboolean rspamd_raw_header_exists(struct rspamd_task *task,
+										 GArray *args,
+										 void *unused);
+static gboolean rspamd_check_smtp_data(struct rspamd_task *task,
+									   GArray *args,
+									   void *unused);
+static gboolean rspamd_content_type_is_type(struct rspamd_task *task,
+											GArray *args,
+											void *unused);
+static gboolean rspamd_content_type_is_subtype(struct rspamd_task *task,
+											   GArray *args,
+											   void *unused);
+static gboolean rspamd_content_type_has_param(struct rspamd_task *task,
+											  GArray *args,
+											  void *unused);
+static gboolean rspamd_content_type_compare_param(struct rspamd_task *task,
+												  GArray *args,
+												  void *unused);
+static gboolean rspamd_has_content_part(struct rspamd_task *task,
+										GArray *args,
+										void *unused);
+static gboolean rspamd_has_content_part_len(struct rspamd_task *task,
+											GArray *args,
+											void *unused);
+static gboolean rspamd_is_empty_body(struct rspamd_task *task,
+									 GArray *args,
+									 void *unused);
+static gboolean rspamd_has_flag_expr(struct rspamd_task *task,
+									 GArray *args,
+									 void *unused);
+static gboolean rspamd_has_symbol_expr(struct rspamd_task *task,
+									   GArray *args,
+									   void *unused);
+
+static rspamd_expression_atom_t *rspamd_mime_expr_parse(const gchar *line, gsize len,
+														rspamd_mempool_t *pool, gpointer ud, GError **err);
+static gdouble rspamd_mime_expr_process(void *ud, rspamd_expression_atom_t *atom);
+static gint rspamd_mime_expr_priority(rspamd_expression_atom_t *atom);
+static void rspamd_mime_expr_destroy(rspamd_expression_atom_t *atom);
+
+/**
+ * Regexp structure
+ */
+struct rspamd_regexp_atom {
+	enum rspamd_re_type type; /**< regexp type										*/
+	gchar *regexp_text;       /**< regexp text representation							*/
+	rspamd_regexp_t *regexp;  /**< regexp structure									*/
+	union {
+		const gchar *header;   /**< header name for header regexps						*/
+		const gchar *selector; /**< selector name for lua selector regexp				*/
+	} extra;
+	gboolean is_test;     /**< true if this expression must be tested				*/
+	gboolean is_strong;   /**< true if headers search must be case sensitive		*/
+	gboolean is_multiple; /**< true if we need to match all inclusions of atom	*/
+};
+
+/**
+ * Rspamd expression function
+ */
+struct rspamd_function_atom {
+	gchar *name;  /**< name of function								*/
+	GArray *args; /**< its args										*/
+};
+
+enum rspamd_mime_atom_type {
+	MIME_ATOM_REGEXP = 0,
+	MIME_ATOM_INTERNAL_FUNCTION,
+	MIME_ATOM_LUA_FUNCTION,
+	MIME_ATOM_LOCAL_LUA_FUNCTION, /* New style */
+};
+
+struct rspamd_mime_atom {
+	gchar *str;
+	union {
+		struct rspamd_regexp_atom *re;
+		struct rspamd_function_atom *func;
+		const gchar *lua_function;
+		gint lua_cbref;
+	} d;
+	enum rspamd_mime_atom_type type;
+};
+
+/*
+ * List of internal functions of rspamd
+ * Sorted by name to use bsearch
+ */
+static struct _fl {
+	const gchar *name;
+	rspamd_internal_func_t func;
+	void *user_data;
+} rspamd_functions_list[] = {
+	{"check_smtp_data", rspamd_check_smtp_data, NULL},
+	{"compare_encoding", rspamd_compare_encoding, NULL},
+	{"compare_parts_distance", rspamd_parts_distance, NULL},
+	{"compare_recipients_distance", rspamd_recipients_distance, NULL},
+	{"compare_transfer_encoding", rspamd_compare_transfer_encoding, NULL},
+	{"content_type_compare_param", rspamd_content_type_compare_param, NULL},
+	{"content_type_has_param", rspamd_content_type_has_param, NULL},
+	{"content_type_is_subtype", rspamd_content_type_is_subtype, NULL},
+	{"content_type_is_type", rspamd_content_type_is_type, NULL},
+	{"has_content_part", rspamd_has_content_part, NULL},
+	{"has_content_part_len", rspamd_has_content_part_len, NULL},
+	{"has_fake_html", rspamd_has_fake_html, NULL},
+	{"has_flag", rspamd_has_flag_expr, NULL},
+	{"has_html_tag", rspamd_has_html_tag, NULL},
+	{"has_only_html_part", rspamd_has_only_html_part, NULL},
+	{"has_symbol", rspamd_has_symbol_expr, NULL},
+	{"header_exists", rspamd_header_exists, NULL},
+	{"is_empty_body", rspamd_is_empty_body, NULL},
+	{"is_html_balanced", rspamd_is_html_balanced, NULL},
+	{"is_recipients_sorted", rspamd_is_recipients_sorted, NULL},
+	{"raw_header_exists", rspamd_raw_header_exists, NULL},
+};
+
+const struct rspamd_atom_subr mime_expr_subr = {
+	.parse = rspamd_mime_expr_parse,
+	.process = rspamd_mime_expr_process,
+	.priority = rspamd_mime_expr_priority,
+	.destroy = rspamd_mime_expr_destroy};
+
+static struct _fl *list_ptr = &rspamd_functions_list[0];
+static guint32 functions_number = sizeof(rspamd_functions_list) /
+								  sizeof(struct _fl);
+static gboolean list_allocated = FALSE;
+
+/* Bsearch routine */
+static gint
+fl_cmp(const void *s1, const void *s2)
+{
+	struct _fl *fl1 = (struct _fl *) s1;
+	struct _fl *fl2 = (struct _fl *) s2;
+	return strcmp(fl1->name, fl2->name);
+}
+
+static GQuark
+rspamd_mime_expr_quark(void)
+{
+	return g_quark_from_static_string("mime-expressions");
+}
+
+#define TYPE_CHECK(str, type, len) (sizeof(type) - 1 == (len) && rspamd_lc_cmp((str), (type), (len)) == 0)
+static gboolean
+rspamd_parse_long_option(const gchar *start, gsize len,
+						 struct rspamd_regexp_atom *a)
+{
+	gboolean ret = FALSE;
+
+	if (TYPE_CHECK(start, "body", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_BODY;
+	}
+	else if (TYPE_CHECK(start, "part", len) ||
+			 TYPE_CHECK(start, "mime", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_MIME;
+	}
+	else if (TYPE_CHECK(start, "raw_part", len) ||
+			 TYPE_CHECK(start, "raw_mime", len) ||
+			 TYPE_CHECK(start, "mime_raw", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_RAWMIME;
+	}
+	else if (TYPE_CHECK(start, "header", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_HEADER;
+	}
+	else if (TYPE_CHECK(start, "mime_header", len) ||
+			 TYPE_CHECK(start, "header_mime", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_MIMEHEADER;
+	}
+	else if (TYPE_CHECK(start, "raw_header", len) ||
+			 TYPE_CHECK(start, "header_raw", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_RAWHEADER;
+	}
+	else if (TYPE_CHECK(start, "all_header", len) ||
+			 TYPE_CHECK(start, "header_all", len) ||
+			 TYPE_CHECK(start, "all_headers", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_ALLHEADER;
+	}
+	else if (TYPE_CHECK(start, "url", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_URL;
+	}
+	else if (TYPE_CHECK(start, "email", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_EMAIL;
+	}
+	else if (TYPE_CHECK(start, "sa_body", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_SABODY;
+	}
+	else if (TYPE_CHECK(start, "sa_raw_body", len) ||
+			 TYPE_CHECK(start, "sa_body_raw", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_SARAWBODY;
+	}
+	else if (TYPE_CHECK(start, "words", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_WORDS;
+	}
+	else if (TYPE_CHECK(start, "raw_words", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_RAWWORDS;
+	}
+	else if (TYPE_CHECK(start, "stem_words", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_STEMWORDS;
+	}
+	else if (TYPE_CHECK(start, "selector", len)) {
+		ret = TRUE;
+		a->type = RSPAMD_RE_SELECTOR;
+	}
+
+	return ret;
+}
+
+/*
+ * Rspamd regexp utility functions
+ */
+static struct rspamd_regexp_atom *
+rspamd_mime_expr_parse_regexp_atom(rspamd_mempool_t *pool, const gchar *line,
+								   struct rspamd_config *cfg)
+{
+	const gchar *begin, *end, *p, *src, *start, *brace;
+	gchar *dbegin, *dend, *extra = NULL;
+	struct rspamd_regexp_atom *result;
+	GError *err = NULL;
+	GString *re_flags;
+
+	if (line == NULL) {
+		msg_err_pool("cannot parse NULL line");
+		return NULL;
+	}
+
+	src = line;
+	result = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_regexp_atom));
+	/* Skip whitespaces */
+	while (g_ascii_isspace(*line)) {
+		line++;
+	}
+	if (*line == '\0') {
+		msg_warn_pool("got empty regexp");
+		return NULL;
+	}
+
+	result->type = RSPAMD_RE_MAX;
+
+	start = line;
+	/* First try to find header name */
+	begin = strchr(line, '/');
+	if (begin != NULL) {
+		p = begin;
+		end = NULL;
+		while (p != line) {
+			if (*p == '=') {
+				end = p;
+				break;
+			}
+			p--;
+		}
+
+		if (end) {
+			extra = rspamd_mempool_alloc(pool, end - line + 1);
+			rspamd_strlcpy(extra, line, end - line + 1);
+			line = end;
+		}
+	}
+	else {
+		extra = rspamd_mempool_strdup(pool, line);
+		result->type = RSPAMD_RE_MAX;
+		line = start;
+	}
+	/* Find begin of regexp */
+	while (*line && *line != '/') {
+		line++;
+	}
+	if (*line != '\0') {
+		begin = line + 1;
+	}
+	else if (extra == NULL) {
+		/* Assume that line without // is just a header name */
+		extra = rspamd_mempool_strdup(pool, line);
+		result->type = RSPAMD_RE_HEADER;
+		return result;
+	}
+	else {
+		/* We got header name earlier but have not found // expression, so it is invalid regexp */
+		msg_warn_pool(
+			"got no header name (eg. header=) but without corresponding regexp, %s",
+			src);
+		return NULL;
+	}
+	/* Find end */
+	end = begin;
+	while (*end && (*end != '/' || *(end - 1) == '\\')) {
+		end++;
+	}
+	if (end == begin || *end != '/') {
+		msg_warn_pool("no trailing / in regexp %s", src);
+		return NULL;
+	}
+	/* Parse flags */
+	p = end + 1;
+	re_flags = g_string_sized_new(32);
+
+	while (p != NULL) {
+		switch (*p) {
+		case 'i':
+		case 'm':
+		case 's':
+		case 'x':
+		case 'u':
+		case 'O':
+		case 'r':
+		case 'L':
+			/* Handled by rspamd_regexp_t */
+			g_string_append_c(re_flags, *p);
+			p++;
+			break;
+		case 'o':
+			p++;
+			break;
+		/* Type flags */
+		case 'H':
+			result->type = RSPAMD_RE_HEADER;
+			p++;
+			break;
+		case 'R':
+			result->type = RSPAMD_RE_ALLHEADER;
+			p++;
+			break;
+		case 'B':
+			result->type = RSPAMD_RE_MIMEHEADER;
+			p++;
+			break;
+		case 'C':
+			result->type = RSPAMD_RE_SABODY;
+			p++;
+			break;
+		case 'D':
+			result->type = RSPAMD_RE_SARAWBODY;
+			p++;
+			break;
+		case 'M':
+			result->type = RSPAMD_RE_BODY;
+			p++;
+			break;
+		case 'P':
+			result->type = RSPAMD_RE_MIME;
+			p++;
+			break;
+		case 'Q':
+			result->type = RSPAMD_RE_RAWMIME;
+			p++;
+			break;
+		case 'U':
+			result->type = RSPAMD_RE_URL;
+			p++;
+			break;
+		case 'X':
+			result->type = RSPAMD_RE_RAWHEADER;
+			p++;
+			break;
+		case '$':
+			result->type = RSPAMD_RE_SELECTOR;
+			p++;
+			break;
+		case '{':
+			/* Long definition */
+			if ((brace = strchr(p + 1, '}')) != NULL) {
+				if (!rspamd_parse_long_option(p + 1, brace - (p + 1), result)) {
+					msg_warn_pool("invalid long regexp type: %*s in '%s'",
+								  (int) (brace - (p + 1)), p + 1, src);
+					p = NULL;
+				}
+				else {
+					p = brace + 1;
+				}
+			}
+			else {
+				p = NULL;
+			}
+			break;
+		/* Other flags */
+		case 'T':
+			result->is_test = TRUE;
+			p++;
+			break;
+		case 'S':
+			result->is_strong = TRUE;
+			p++;
+			break;
+		case 'A':
+			result->is_multiple = TRUE;
+			p++;
+			break;
+		/* Stop flags parsing */
+		default:
+			p = NULL;
+			break;
+		}
+	}
+
+	if (result->type >= RSPAMD_RE_MAX) {
+		if (extra) {
+			/* Assume header regexp */
+			result->extra.header = extra;
+			result->type = RSPAMD_RE_HEADER;
+		}
+		else {
+			msg_err_pool("could not read regexp: %s, unknown type", src);
+			return NULL;
+		}
+	}
+
+	if ((result->type == RSPAMD_RE_HEADER ||
+		 result->type == RSPAMD_RE_RAWHEADER ||
+		 result->type == RSPAMD_RE_MIMEHEADER)) {
+		if (extra == NULL) {
+			msg_err_pool("header regexp: '%s' has no header part", src);
+			return NULL;
+		}
+		else {
+			result->extra.header = extra;
+		}
+	}
+
+	if (result->type == RSPAMD_RE_SELECTOR) {
+		if (extra == NULL) {
+			msg_err_pool("selector regexp: '%s' has no selector part", src);
+			return NULL;
+		}
+		else {
+			result->extra.selector = extra;
+		}
+	}
+
+
+	result->regexp_text = rspamd_mempool_strdup(pool, start);
+	dbegin = result->regexp_text + (begin - start);
+	dend = result->regexp_text + (end - start);
+	*dend = '\0';
+
+	result->regexp = rspamd_regexp_new(dbegin, re_flags->str,
+									   &err);
+
+	g_string_free(re_flags, TRUE);
+
+	if (result->regexp == NULL || err != NULL) {
+		msg_warn_pool("could not read regexp: %s while reading regexp %e",
+					  src, err);
+
+		if (err) {
+			g_error_free(err);
+		}
+
+		return NULL;
+	}
+
+	if (result->is_multiple) {
+		rspamd_regexp_set_maxhits(result->regexp, 0);
+	}
+	else {
+		rspamd_regexp_set_maxhits(result->regexp, 1);
+	}
+
+	rspamd_regexp_set_ud(result->regexp, result);
+
+	*dend = '/';
+
+	return result;
+}
+
+struct rspamd_function_atom *
+rspamd_mime_expr_parse_function_atom(rspamd_mempool_t *pool, const gchar *input)
+{
+	const gchar *obrace, *ebrace, *p, *c;
+	gchar t, *databuf;
+	guint len;
+	struct rspamd_function_atom *res;
+	struct expression_argument arg;
+	GError *err = NULL;
+	enum {
+		start_read_argument = 0,
+		in_string,
+		in_regexp,
+		got_backslash,
+		got_comma
+	} state,
+		prev_state = 0;
+
+	obrace = strchr(input, '(');
+	ebrace = strrchr(input, ')');
+
+	g_assert(obrace != NULL && ebrace != NULL);
+
+	res = rspamd_mempool_alloc0(pool, sizeof(*res));
+	res->name = rspamd_mempool_alloc(pool, obrace - input + 1);
+	rspamd_strlcpy(res->name, input, obrace - input + 1);
+	res->args = g_array_new(FALSE, FALSE, sizeof(struct expression_argument));
+
+	p = obrace + 1;
+	c = p;
+	state = start_read_argument;
+
+	/* Read arguments */
+	while (p <= ebrace) {
+		t = *p;
+		switch (state) {
+		case start_read_argument:
+			if (t == '/') {
+				state = in_regexp;
+				c = p;
+			}
+			else if (!g_ascii_isspace(t)) {
+				state = in_string;
+
+				if (t == '\'' || t == '\"') {
+					c = p + 1;
+				}
+				else {
+					c = p;
+				}
+			}
+			p++;
+			break;
+		case in_regexp:
+			if (t == '\\') {
+				state = got_backslash;
+				prev_state = in_regexp;
+			}
+			else if (t == ',' || p == ebrace) {
+				len = p - c + 1;
+				databuf = rspamd_mempool_alloc(pool, len);
+				rspamd_strlcpy(databuf, c, len);
+				arg.type = EXPRESSION_ARGUMENT_REGEXP;
+				arg.data = rspamd_regexp_cache_create(NULL, databuf, NULL, &err);
+
+				if (arg.data == NULL) {
+					/* Fallback to string */
+					msg_warn("cannot parse slashed argument %s as regexp: %s",
+							 databuf, err->message);
+					g_error_free(err);
+					arg.type = EXPRESSION_ARGUMENT_NORMAL;
+					arg.data = databuf;
+				}
+
+				g_array_append_val(res->args, arg);
+				state = got_comma;
+			}
+			p++;
+			break;
+		case in_string:
+			if (t == '\\') {
+				state = got_backslash;
+				prev_state = in_string;
+			}
+			else if (t == ',' || p == ebrace) {
+				if (*(p - 1) == '\'' || *(p - 1) == '\"') {
+					len = p - c;
+				}
+				else {
+					len = p - c + 1;
+				}
+
+				databuf = rspamd_mempool_alloc(pool, len);
+				rspamd_strlcpy(databuf, c, len);
+				arg.type = EXPRESSION_ARGUMENT_NORMAL;
+				arg.data = databuf;
+				g_array_append_val(res->args, arg);
+				state = got_comma;
+			}
+			p++;
+			break;
+		case got_backslash:
+			state = prev_state;
+			p++;
+			break;
+		case got_comma:
+			state = start_read_argument;
+			break;
+		}
+	}
+
+	return res;
+}
+
+static rspamd_expression_atom_t *
+rspamd_mime_expr_parse(const gchar *line, gsize len,
+					   rspamd_mempool_t *pool, gpointer ud, GError **err)
+{
+	rspamd_expression_atom_t *a = NULL;
+	struct rspamd_mime_atom *mime_atom = NULL;
+	const gchar *p, *end, *c = NULL;
+	struct rspamd_mime_expr_ud *real_ud = (struct rspamd_mime_expr_ud *) ud;
+	struct rspamd_config *cfg;
+	rspamd_regexp_t *own_re;
+	gchar t;
+	gint type = MIME_ATOM_REGEXP, obraces = 0, ebraces = 0;
+	enum {
+		in_header = 0,
+		got_slash,
+		in_regexp,
+		got_backslash,
+		got_second_slash,
+		in_flags,
+		in_flags_brace,
+		got_obrace,
+		in_function,
+		in_local_function,
+		got_ebrace,
+		end_atom,
+		bad_atom
+	} state = 0,
+	  prev_state = 0;
+
+	p = line;
+	end = p + len;
+	cfg = real_ud->cfg;
+
+	while (p < end) {
+		t = *p;
+
+		switch (state) {
+		case in_header:
+			if (t == '/') {
+				/* Regexp */
+				state = got_slash;
+			}
+			else if (t == '(') {
+				/* Function */
+				state = got_obrace;
+			}
+			else if (!g_ascii_isalnum(t) && t != '_' && t != '-' && t != '=') {
+				if (t == ':') {
+					if (p - line == 3 && memcmp(line, "lua", 3) == 0) {
+						type = MIME_ATOM_LOCAL_LUA_FUNCTION;
+						state = in_local_function;
+						c = p + 1;
+					}
+				}
+				else {
+					/* Likely lua function, identified by just a string */
+					type = MIME_ATOM_LUA_FUNCTION;
+					state = end_atom;
+					/* Do not increase p */
+					continue;
+				}
+			}
+			else if (g_ascii_isspace(t)) {
+				state = bad_atom;
+			}
+			p++;
+			break;
+		case got_slash:
+			state = in_regexp;
+			break;
+		case in_regexp:
+			if (t == '\\') {
+				state = got_backslash;
+				prev_state = in_regexp;
+			}
+			else if (t == '/') {
+				state = got_second_slash;
+			}
+			p++;
+			break;
+		case got_second_slash:
+			state = in_flags;
+			break;
+		case in_flags:
+			if (t == '{') {
+				state = in_flags_brace;
+				p++;
+			}
+			else if (!g_ascii_isalpha(t) && t != '$') {
+				state = end_atom;
+			}
+			else {
+				p++;
+			}
+			break;
+		case in_flags_brace:
+			if (t == '}') {
+				state = in_flags;
+			}
+			p++;
+			break;
+		case got_backslash:
+			state = prev_state;
+			p++;
+			break;
+		case got_obrace:
+			state = in_function;
+			type = MIME_ATOM_INTERNAL_FUNCTION;
+			obraces++;
+			break;
+		case in_function:
+			if (t == '\\') {
+				state = got_backslash;
+				prev_state = in_function;
+			}
+			else if (t == '(') {
+				obraces++;
+			}
+			else if (t == ')') {
+				ebraces++;
+				if (ebraces == obraces) {
+					state = got_ebrace;
+				}
+			}
+			p++;
+			break;
+		case in_local_function:
+			if (!(g_ascii_isalnum(t) || t == '-' || t == '_')) {
+				g_assert(c != NULL);
+				state = end_atom;
+			}
+			else {
+				p++;
+			}
+			break;
+		case got_ebrace:
+			state = end_atom;
+			break;
+		case bad_atom:
+			g_set_error(err, rspamd_mime_expr_quark(), 100, "cannot parse"
+															" mime atom '%s' when reading symbol '%c' at offset %d, "
+															"near %.*s",
+						line, t, (gint) (p - line),
+						(gint) MIN(end - p, 10), p);
+			return NULL;
+		case end_atom:
+			goto set;
+		}
+	}
+set:
+
+	if (p - line == 0 || (state != got_ebrace && state != got_second_slash &&
+						  state != in_flags && state != end_atom)) {
+		g_set_error(err, rspamd_mime_expr_quark(), 200, "incomplete or empty"
+														" mime atom");
+		return NULL;
+	}
+
+	mime_atom = rspamd_mempool_alloc(pool, sizeof(*mime_atom));
+	mime_atom->type = type;
+	mime_atom->str = rspamd_mempool_alloc(pool, p - line + 1);
+	rspamd_strlcpy(mime_atom->str, line, p - line + 1);
+
+	if (type == MIME_ATOM_REGEXP) {
+		mime_atom->d.re = rspamd_mime_expr_parse_regexp_atom(pool,
+															 mime_atom->str, cfg);
+		if (mime_atom->d.re == NULL) {
+			g_set_error(err, rspamd_mime_expr_quark(), 200,
+						"cannot parse regexp '%s'",
+						mime_atom->str);
+			goto err;
+		}
+		else {
+			gint lua_cbref = -1;
+
+			/* Check regexp condition */
+			if (real_ud->conf_obj != NULL) {
+				const ucl_object_t *re_conditions = ucl_object_lookup(real_ud->conf_obj,
+																	  "re_conditions");
+
+				if (re_conditions != NULL) {
+					if (ucl_object_type(re_conditions) != UCL_OBJECT) {
+						g_set_error(err, rspamd_mime_expr_quark(), 320,
+									"re_conditions is not a table for '%s'",
+									mime_atom->str);
+						rspamd_regexp_unref(mime_atom->d.re->regexp);
+						goto err;
+					}
+
+					const ucl_object_t *function_obj = ucl_object_lookup(re_conditions,
+																		 mime_atom->str);
+
+					if (function_obj != NULL) {
+						if (ucl_object_type(function_obj) != UCL_USERDATA) {
+							g_set_error(err, rspamd_mime_expr_quark(), 320,
+										"condition for '%s' is invalid, must be function",
+										mime_atom->str);
+							rspamd_regexp_unref(mime_atom->d.re->regexp);
+							goto err;
+						}
+
+						struct ucl_lua_funcdata *fd = function_obj->value.ud;
+
+						lua_cbref = fd->idx;
+					}
+				}
+			}
+
+			if (lua_cbref != -1) {
+				msg_info_config("added condition for regexp %s", mime_atom->str);
+				/* Add SOM_LEFTMOST_FLAG implicitly */
+				rspamd_regexp_set_flags(mime_atom->d.re->regexp, rspamd_regexp_get_flags(mime_atom->d.re->regexp) |
+																	 RSPAMD_REGEXP_FLAG_LEFTMOST);
+			}
+
+			/* Register new item in the cache */
+			if (mime_atom->d.re->type == RSPAMD_RE_HEADER ||
+				mime_atom->d.re->type == RSPAMD_RE_RAWHEADER ||
+				mime_atom->d.re->type == RSPAMD_RE_MIMEHEADER) {
+
+				if (mime_atom->d.re->extra.header != NULL) {
+					own_re = mime_atom->d.re->regexp;
+					mime_atom->d.re->regexp = rspamd_re_cache_add(cfg->re_cache,
+																  mime_atom->d.re->regexp,
+																  mime_atom->d.re->type,
+																  mime_atom->d.re->extra.header,
+																  strlen(mime_atom->d.re->extra.header) + 1,
+																  lua_cbref);
+					/* Pass ownership to the cache */
+					rspamd_regexp_unref(own_re);
+				}
+				else {
+					/* We have header regexp, but no header name is detected */
+					g_set_error(err,
+								rspamd_mime_expr_quark(),
+								200,
+								"no header name in header regexp: '%s'",
+								mime_atom->str);
+					rspamd_regexp_unref(mime_atom->d.re->regexp);
+					goto err;
+				}
+			}
+			else if (mime_atom->d.re->type == RSPAMD_RE_SELECTOR) {
+				if (mime_atom->d.re->extra.selector != NULL) {
+					own_re = mime_atom->d.re->regexp;
+					mime_atom->d.re->regexp = rspamd_re_cache_add(cfg->re_cache,
+																  mime_atom->d.re->regexp,
+																  mime_atom->d.re->type,
+																  mime_atom->d.re->extra.selector,
+																  strlen(mime_atom->d.re->extra.selector) + 1,
+																  lua_cbref);
+					/* Pass ownership to the cache */
+					rspamd_regexp_unref(own_re);
+				}
+				else {
+					/* We have selector regexp, but no selector name is detected */
+					g_set_error(err,
+								rspamd_mime_expr_quark(),
+								200,
+								"no selector name in selector regexp: '%s'",
+								mime_atom->str);
+					rspamd_regexp_unref(mime_atom->d.re->regexp);
+					goto err;
+				}
+			}
+			else {
+				own_re = mime_atom->d.re->regexp;
+				mime_atom->d.re->regexp = rspamd_re_cache_add(cfg->re_cache,
+															  mime_atom->d.re->regexp,
+															  mime_atom->d.re->type,
+															  NULL,
+															  0,
+															  lua_cbref);
+				/* Pass ownership to the cache */
+				rspamd_regexp_unref(own_re);
+			}
+		}
+	}
+	else if (type == MIME_ATOM_LUA_FUNCTION) {
+		mime_atom->d.lua_function = mime_atom->str;
+
+		lua_getglobal(cfg->lua_state, mime_atom->str);
+
+		if (lua_type(cfg->lua_state, -1) != LUA_TFUNCTION) {
+			g_set_error(err, rspamd_mime_expr_quark(), 200,
+						"no such lua function '%s'",
+						mime_atom->str);
+			lua_pop(cfg->lua_state, 1);
+
+			goto err;
+		}
+
+		lua_pop(cfg->lua_state, 1);
+	}
+	else if (type == MIME_ATOM_LOCAL_LUA_FUNCTION) {
+		/* p pointer is set to the start of Lua function name */
+
+		if (real_ud->conf_obj == NULL) {
+			g_set_error(err, rspamd_mime_expr_quark(), 300,
+						"no config object for '%s'",
+						mime_atom->str);
+			goto err;
+		}
+
+		const ucl_object_t *functions = ucl_object_lookup(real_ud->conf_obj,
+														  "functions");
+
+		if (functions == NULL) {
+			g_set_error(err, rspamd_mime_expr_quark(), 310,
+						"no functions defined for '%s'",
+						mime_atom->str);
+			goto err;
+		}
+
+		if (ucl_object_type(functions) != UCL_OBJECT) {
+			g_set_error(err, rspamd_mime_expr_quark(), 320,
+						"functions is not a table for '%s'",
+						mime_atom->str);
+			goto err;
+		}
+
+		const ucl_object_t *function_obj;
+
+		function_obj = ucl_object_lookup_len(functions, c,
+											 p - c);
+
+		if (function_obj == NULL) {
+			g_set_error(err, rspamd_mime_expr_quark(), 320,
+						"function %.*s is not found for '%s'",
+						(int) (p - c), c, mime_atom->str);
+			goto err;
+		}
+
+		if (ucl_object_type(function_obj) != UCL_USERDATA) {
+			g_set_error(err, rspamd_mime_expr_quark(), 320,
+						"function %.*s has invalid type for '%s'",
+						(int) (p - c), c, mime_atom->str);
+			goto err;
+		}
+
+		struct ucl_lua_funcdata *fd = function_obj->value.ud;
+
+		mime_atom->d.lua_cbref = fd->idx;
+	}
+	else {
+		mime_atom->d.func = rspamd_mime_expr_parse_function_atom(pool,
+																 mime_atom->str);
+		if (mime_atom->d.func == NULL) {
+			g_set_error(err, rspamd_mime_expr_quark(), 200,
+						"cannot parse function '%s'",
+						mime_atom->str);
+			goto err;
+		}
+	}
+
+	a = rspamd_mempool_alloc0(pool, sizeof(*a));
+	a->len = p - line;
+	a->priority = 0;
+	a->data = mime_atom;
+
+	return a;
+
+err:
+
+	return NULL;
+}
+
+static gint
+rspamd_mime_expr_process_regexp(struct rspamd_regexp_atom *re,
+								struct rspamd_task *task)
+{
+	gint ret;
+
+	if (re == NULL) {
+		msg_info_task("invalid regexp passed");
+		return 0;
+	}
+
+	if (re->type == RSPAMD_RE_HEADER || re->type == RSPAMD_RE_RAWHEADER) {
+		ret = rspamd_re_cache_process(task,
+									  re->regexp,
+									  re->type,
+									  re->extra.header,
+									  strlen(re->extra.header),
+									  re->is_strong);
+	}
+	else if (re->type == RSPAMD_RE_SELECTOR) {
+		ret = rspamd_re_cache_process(task,
+									  re->regexp,
+									  re->type,
+									  re->extra.selector,
+									  strlen(re->extra.selector),
+									  re->is_strong);
+	}
+	else {
+		ret = rspamd_re_cache_process(task,
+									  re->regexp,
+									  re->type,
+									  NULL,
+									  0,
+									  re->is_strong);
+	}
+
+	if (re->is_test) {
+		msg_info_task("test %s regexp '%s' returned %d",
+					  rspamd_re_cache_type_to_string(re->type),
+					  re->regexp_text, ret);
+	}
+
+	return ret;
+}
+
+
+static gint
+rspamd_mime_expr_priority(rspamd_expression_atom_t *atom)
+{
+	struct rspamd_mime_atom *mime_atom = atom->data;
+	gint ret = 0;
+
+	switch (mime_atom->type) {
+	case MIME_ATOM_INTERNAL_FUNCTION:
+		/* Prioritize internal functions slightly */
+		ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 8;
+		break;
+	case MIME_ATOM_LUA_FUNCTION:
+	case MIME_ATOM_LOCAL_LUA_FUNCTION:
+		ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 4;
+		break;
+	case MIME_ATOM_REGEXP:
+		switch (mime_atom->d.re->type) {
+		case RSPAMD_RE_HEADER:
+		case RSPAMD_RE_RAWHEADER:
+			ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 16;
+			break;
+		case RSPAMD_RE_URL:
+		case RSPAMD_RE_EMAIL:
+			ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 8;
+			break;
+		case RSPAMD_RE_SELECTOR:
+			ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 8;
+			break;
+		case RSPAMD_RE_MIME:
+		case RSPAMD_RE_RAWMIME:
+			ret = RSPAMD_EXPRESSION_MAX_PRIORITY - RSPAMD_EXPRESSION_MAX_PRIORITY / 2;
+			break;
+		case RSPAMD_RE_WORDS:
+		case RSPAMD_RE_RAWWORDS:
+		case RSPAMD_RE_STEMWORDS:
+		default:
+			/* For expensive regexps */
+			ret = 0;
+			break;
+		}
+	}
+
+	return ret;
+}
+
+static void
+rspamd_mime_expr_destroy(rspamd_expression_atom_t *atom)
+{
+	struct rspamd_mime_atom *mime_atom = atom->data;
+
+	if (mime_atom) {
+		if (mime_atom->type == MIME_ATOM_INTERNAL_FUNCTION) {
+			/* Need to cleanup arguments */
+			g_array_free(mime_atom->d.func->args, TRUE);
+		}
+	}
+}
+
+static gboolean
+rspamd_mime_expr_process_function(struct rspamd_function_atom *func,
+								  struct rspamd_task *task,
+								  lua_State *L)
+{
+	struct _fl *selected, key;
+
+	key.name = func->name;
+
+	selected = bsearch(&key,
+					   list_ptr,
+					   functions_number,
+					   sizeof(struct _fl),
+					   fl_cmp);
+	if (selected == NULL) {
+		/* Try to check lua function */
+		return FALSE;
+	}
+
+	return selected->func(task, func->args, selected->user_data);
+}
+
+static gdouble
+rspamd_mime_expr_process(void *ud, rspamd_expression_atom_t *atom)
+{
+	struct rspamd_task *task = (struct rspamd_task *) ud;
+	struct rspamd_mime_atom *mime_atom;
+	lua_State *L;
+	gdouble ret = 0;
+
+	g_assert(task != NULL);
+	g_assert(atom != NULL);
+
+	mime_atom = atom->data;
+
+	if (mime_atom->type == MIME_ATOM_REGEXP) {
+		ret = rspamd_mime_expr_process_regexp(mime_atom->d.re, task);
+	}
+	else if (mime_atom->type == MIME_ATOM_LUA_FUNCTION) {
+		L = task->cfg->lua_state;
+		lua_getglobal(L, mime_atom->d.lua_function);
+		rspamd_lua_task_push(L, task);
+
+		if (lua_pcall(L, 1, 1, 0) != 0) {
+			msg_info_task("lua call to global function '%s' for atom '%s' failed: %s",
+						  mime_atom->d.lua_function,
+						  mime_atom->str,
+						  lua_tostring(L, -1));
+			lua_pop(L, 1);
+		}
+		else {
+			if (lua_type(L, -1) == LUA_TBOOLEAN) {
+				ret = lua_toboolean(L, -1);
+			}
+			else if (lua_type(L, -1) == LUA_TNUMBER) {
+				ret = lua_tonumber(L, 1);
+			}
+			else {
+				msg_err_task("%s returned wrong return type: %s",
+							 mime_atom->str, lua_typename(L, lua_type(L, -1)));
+			}
+			/* Remove result */
+			lua_pop(L, 1);
+		}
+	}
+	else if (mime_atom->type == MIME_ATOM_LOCAL_LUA_FUNCTION) {
+		gint err_idx;
+
+		L = task->cfg->lua_state;
+		lua_pushcfunction(L, &rspamd_lua_traceback);
+		err_idx = lua_gettop(L);
+
+		lua_rawgeti(L, LUA_REGISTRYINDEX, mime_atom->d.lua_cbref);
+		rspamd_lua_task_push(L, task);
+
+		if (lua_pcall(L, 1, 1, err_idx) != 0) {
+			msg_info_task("lua call to local function for atom '%s' failed: %s",
+						  mime_atom->str,
+						  lua_tostring(L, -1));
+		}
+		else {
+			if (lua_type(L, -1) == LUA_TBOOLEAN) {
+				ret = lua_toboolean(L, -1);
+			}
+			else if (lua_type(L, -1) == LUA_TNUMBER) {
+				ret = lua_tonumber(L, 1);
+			}
+			else {
+				msg_err_task("%s returned wrong return type: %s",
+							 mime_atom->str, lua_typename(L, lua_type(L, -1)));
+			}
+		}
+
+		lua_settop(L, 0);
+	}
+	else {
+		ret = rspamd_mime_expr_process_function(mime_atom->d.func, task,
+												task->cfg->lua_state);
+	}
+
+	return ret;
+}
+
+void register_expression_function(const gchar *name,
+								  rspamd_internal_func_t func,
+								  void *user_data)
+{
+	static struct _fl *new;
+
+	functions_number++;
+
+	new = g_new(struct _fl, functions_number);
+	memcpy(new, list_ptr, (functions_number - 1) * sizeof(struct _fl));
+	if (list_allocated) {
+		g_free(list_ptr);
+	}
+
+	list_allocated = TRUE;
+	new[functions_number - 1].name = name;
+	new[functions_number - 1].func = func;
+	new[functions_number - 1].user_data = user_data;
+	qsort(new, functions_number, sizeof(struct _fl), fl_cmp);
+	list_ptr = new;
+}
+
+gboolean
+rspamd_compare_encoding(struct rspamd_task *task, GArray *args, void *unused)
+{
+	struct expression_argument *arg;
+
+	if (args == NULL || task == NULL) {
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+	if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("invalid argument to function is passed");
+		return FALSE;
+	}
+
+	/* XXX: really write this function */
+	return TRUE;
+}
+
+gboolean
+rspamd_header_exists(struct rspamd_task *task, GArray *args, void *unused)
+{
+	struct expression_argument *arg;
+	struct rspamd_mime_header *rh;
+
+	if (args == NULL || task == NULL) {
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+	if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("invalid argument to function is passed");
+		return FALSE;
+	}
+
+	rh = rspamd_message_get_header_array(task,
+										 (gchar *) arg->data, FALSE);
+
+	debug_task("try to get header %s: %d", (gchar *) arg->data,
+			   (rh != NULL));
+
+	if (rh) {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+
+/*
+ * This function is designed to find difference between text/html and text/plain parts
+ * It takes one argument: difference threshold, if we have two text parts, compare
+ * its hashes and check for threshold, if value is greater than threshold, return TRUE
+ * and return FALSE otherwise.
+ */
+gboolean
+rspamd_parts_distance(struct rspamd_task *task, GArray *args, void *unused)
+{
+	gint threshold, threshold2 = -1;
+	struct expression_argument *arg;
+	gdouble *pdiff, diff;
+
+	if (args == NULL || args->len == 0) {
+		debug_task("no threshold is specified, assume it 100");
+		threshold = 100;
+	}
+	else {
+		errno = 0;
+		arg = &g_array_index(args, struct expression_argument, 0);
+		if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+			msg_warn_task("invalid argument to function is passed");
+			return FALSE;
+		}
+
+		threshold = strtoul((gchar *) arg->data, NULL, 10);
+		if (errno != 0) {
+			msg_info_task("bad numeric value for threshold \"%s\", assume it 100",
+						  (gchar *) arg->data);
+			threshold = 100;
+		}
+		if (args->len >= 2) {
+			arg = &g_array_index(args, struct expression_argument, 1);
+			if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+				msg_warn_task("invalid argument to function is passed");
+				return FALSE;
+			}
+
+			errno = 0;
+			threshold2 = strtoul((gchar *) arg->data, NULL, 10);
+			if (errno != 0) {
+				msg_info_task("bad numeric value for threshold \"%s\", ignore it",
+							  (gchar *) arg->data);
+				threshold2 = -1;
+			}
+		}
+	}
+
+	if ((pdiff =
+			 rspamd_mempool_get_variable(task->task_pool,
+										 "parts_distance")) != NULL) {
+		diff = (1.0 - (*pdiff)) * 100.0;
+
+		if (diff != -1) {
+			if (threshold2 > 0) {
+				if (diff >= MIN(threshold, threshold2) &&
+					diff < MAX(threshold, threshold2)) {
+
+					return TRUE;
+				}
+			}
+			else {
+				if (diff <= threshold) {
+					return TRUE;
+				}
+			}
+			return FALSE;
+		}
+		else {
+			return FALSE;
+		}
+	}
+
+	return FALSE;
+}
+
+struct addr_list {
+	const gchar *name;
+	guint namelen;
+	const gchar *addr;
+	guint addrlen;
+};
+
+static gint
+addr_list_cmp_func(const void *a, const void *b)
+{
+	const struct addr_list *addra = (struct addr_list *) a,
+						   *addrb = (struct addr_list *) b;
+
+	if (addra->addrlen != addrb->addrlen) {
+		return addra->addrlen - addrb->addrlen;
+	}
+
+	return memcmp(addra->addr, addrb->addr, addra->addrlen);
+}
+
+#define COMPARE_RCPT_LEN 3
+#define MIN_RCPT_TO_COMPARE 7
+
+gboolean
+rspamd_recipients_distance(struct rspamd_task *task, GArray *args,
+						   void *unused)
+{
+	struct expression_argument *arg;
+	struct rspamd_email_address *cur;
+	double threshold;
+	struct addr_list *ar;
+	gint num, i, hits = 0;
+
+	if (args == NULL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+	if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("invalid argument to function is passed");
+		return FALSE;
+	}
+
+	errno = 0;
+	threshold = strtod((gchar *) arg->data, NULL);
+
+	if (errno != 0) {
+		msg_warn_task("invalid numeric value '%s': %s",
+					  (gchar *) arg->data,
+					  strerror(errno));
+		return FALSE;
+	}
+
+	if (!MESSAGE_FIELD(task, rcpt_mime)) {
+		return FALSE;
+	}
+
+	num = MESSAGE_FIELD(task, rcpt_mime)->len;
+
+	if (num < MIN_RCPT_TO_COMPARE) {
+		return FALSE;
+	}
+
+	ar = rspamd_mempool_alloc0(task->task_pool, num * sizeof(struct addr_list));
+
+	/* Fill array */
+	num = 0;
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, rcpt_mime), i, cur)
+	{
+		if (cur->addr_len > COMPARE_RCPT_LEN) {
+			ar[num].name = cur->addr;
+			ar[num].namelen = cur->addr_len;
+			ar[num].addr = cur->domain;
+			ar[num].addrlen = cur->domain_len;
+			num++;
+		}
+	}
+
+	qsort(ar, num, sizeof(*ar), addr_list_cmp_func);
+
+	/* Cycle all elements in array */
+	for (i = 0; i < num; i++) {
+		if (i < num - 1) {
+			if (ar[i].namelen == ar[i + 1].namelen) {
+				if (rspamd_lc_cmp(ar[i].name, ar[i + 1].name, COMPARE_RCPT_LEN) == 0) {
+					hits++;
+				}
+			}
+		}
+	}
+
+	if ((hits * num / 2.) / (double) num >= threshold) {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+gboolean
+rspamd_has_only_html_part(struct rspamd_task *task, GArray *args,
+						  void *unused)
+{
+	struct rspamd_mime_text_part *p;
+	guint i, cnt_html = 0, cnt_txt = 0;
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, p)
+	{
+		if (!IS_TEXT_PART_ATTACHMENT(p)) {
+			if (IS_TEXT_PART_HTML(p)) {
+				cnt_html++;
+			}
+			else {
+				cnt_txt++;
+			}
+		}
+	}
+
+	return (cnt_html > 0 && cnt_txt == 0);
+}
+
+static gboolean
+is_recipient_list_sorted(GPtrArray *ar)
+{
+	struct rspamd_email_address *addr;
+	gboolean res = TRUE;
+	rspamd_ftok_t cur, prev;
+	gint i;
+
+	/* Do not check to short address lists */
+	if (ar == NULL || ar->len < MIN_RCPT_TO_COMPARE) {
+		return FALSE;
+	}
+
+	prev.len = 0;
+	prev.begin = NULL;
+
+	PTR_ARRAY_FOREACH(ar, i, addr)
+	{
+		cur.begin = addr->addr;
+		cur.len = addr->addr_len;
+
+		if (prev.len != 0) {
+			if (rspamd_ftok_casecmp(&cur, &prev) <= 0) {
+				res = FALSE;
+				break;
+			}
+		}
+
+		prev = cur;
+	}
+
+	return res;
+}
+
+gboolean
+rspamd_is_recipients_sorted(struct rspamd_task *task,
+							GArray *args,
+							void *unused)
+{
+	/* Check all types of addresses */
+
+	if (MESSAGE_FIELD(task, rcpt_mime)) {
+		return is_recipient_list_sorted(MESSAGE_FIELD(task, rcpt_mime));
+	}
+
+	return FALSE;
+}
+
+gboolean
+rspamd_compare_transfer_encoding(struct rspamd_task *task,
+								 GArray *args,
+								 void *unused)
+{
+	struct expression_argument *arg;
+	guint i;
+	struct rspamd_mime_part *part;
+	enum rspamd_cte cte;
+
+	if (args == NULL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+	if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("invalid argument to function is passed");
+		return FALSE;
+	}
+
+	cte = rspamd_cte_from_string(arg->data);
+
+	if (cte == RSPAMD_CTE_UNKNOWN) {
+		msg_warn_task("unknown cte: %s", arg->data);
+		return FALSE;
+	}
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+	{
+		if (IS_PART_TEXT(part)) {
+			if (part->cte == cte) {
+				return TRUE;
+			}
+		}
+	}
+
+	return FALSE;
+}
+
+gboolean
+rspamd_is_html_balanced(struct rspamd_task *task, GArray *args, void *unused)
+{
+	/* Totally broken but seems to be never used */
+	return TRUE;
+}
+
+gboolean
+rspamd_has_html_tag(struct rspamd_task *task, GArray *args, void *unused)
+{
+	struct rspamd_mime_text_part *p;
+	struct expression_argument *arg;
+	guint i;
+	gboolean res = FALSE;
+
+	if (args == NULL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+	if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("invalid argument to function is passed");
+		return FALSE;
+	}
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, p)
+	{
+		if (IS_TEXT_PART_HTML(p) && p->html) {
+			res = rspamd_html_tag_seen(p->html, arg->data);
+		}
+
+		if (res) {
+			break;
+		}
+	}
+
+	return res;
+}
+
+gboolean
+rspamd_has_fake_html(struct rspamd_task *task, GArray *args, void *unused)
+{
+	struct rspamd_mime_text_part *p;
+	guint i;
+	gboolean res = FALSE;
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, p)
+	{
+		if (IS_TEXT_PART_HTML(p) && (rspamd_html_get_tags_count(p->html) < 2)) {
+			res = TRUE;
+		}
+
+		if (res) {
+			break;
+		}
+	}
+
+	return res;
+}
+
+static gboolean
+rspamd_raw_header_exists(struct rspamd_task *task, GArray *args, void *unused)
+{
+	struct expression_argument *arg;
+
+	if (args == NULL || task == NULL) {
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+	if (!arg || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("invalid argument to function is passed");
+		return FALSE;
+	}
+
+	return rspamd_message_get_header_array(task, arg->data, FALSE) != NULL;
+}
+
+static gboolean
+match_smtp_data(struct rspamd_task *task,
+				struct expression_argument *arg,
+				const gchar *what, gsize len)
+{
+	rspamd_regexp_t *re;
+	gint r = 0;
+
+	if (arg->type == EXPRESSION_ARGUMENT_REGEXP) {
+		/* This is a regexp */
+		re = arg->data;
+		if (re == NULL) {
+			msg_warn_task("cannot compile regexp for function");
+			return FALSE;
+		}
+
+
+		if (len > 0) {
+			r = rspamd_regexp_search(re, what, len, NULL, NULL, FALSE, NULL);
+		}
+
+		return r;
+	}
+	else if (arg->type == EXPRESSION_ARGUMENT_NORMAL &&
+			 g_ascii_strncasecmp(arg->data, what, len) == 0) {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static gboolean
+rspamd_check_smtp_data(struct rspamd_task *task, GArray *args, void *unused)
+{
+	struct expression_argument *arg;
+	struct rspamd_email_address *addr = NULL;
+	GPtrArray *rcpts = NULL;
+	const gchar *type, *str = NULL;
+	guint i;
+
+	if (args == NULL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+
+	if (!arg || !arg->data || arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+	else {
+		type = arg->data;
+		switch (*type) {
+		case 'f':
+		case 'F':
+			if (g_ascii_strcasecmp(type, "from") == 0) {
+				addr = rspamd_task_get_sender(task);
+			}
+			else {
+				msg_warn_task("bad argument to function: %s", type);
+				return FALSE;
+			}
+			break;
+		case 'h':
+		case 'H':
+			if (g_ascii_strcasecmp(type, "helo") == 0) {
+				str = task->helo;
+			}
+			else {
+				msg_warn_task("bad argument to function: %s", type);
+				return FALSE;
+			}
+			break;
+		case 'u':
+		case 'U':
+			if (g_ascii_strcasecmp(type, "user") == 0) {
+				str = task->auth_user;
+			}
+			else {
+				msg_warn_task("bad argument to function: %s", type);
+				return FALSE;
+			}
+			break;
+		case 's':
+		case 'S':
+			if (g_ascii_strcasecmp(type, "subject") == 0) {
+				str = MESSAGE_FIELD(task, subject);
+			}
+			else {
+				msg_warn_task("bad argument to function: %s", type);
+				return FALSE;
+			}
+			break;
+		case 'r':
+		case 'R':
+			if (g_ascii_strcasecmp(type, "rcpt") == 0) {
+				rcpts = task->rcpt_envelope;
+			}
+			else {
+				msg_warn_task("bad argument to function: %s", type);
+				return FALSE;
+			}
+			break;
+		default:
+			msg_warn_task("bad argument to function: %s", type);
+			return FALSE;
+		}
+	}
+
+	if (str == NULL && addr == NULL && rcpts == NULL) {
+		/* Not enough data so regexp would NOT be found anyway */
+		return FALSE;
+	}
+
+	/* We would process only one more argument, others are ignored */
+	if (args->len >= 2) {
+		arg = &g_array_index(args, struct expression_argument, 1);
+
+		if (arg) {
+			if (str != NULL) {
+				return match_smtp_data(task, arg, str, strlen(str));
+			}
+			else if (addr != NULL && addr->addr) {
+				return match_smtp_data(task, arg, addr->addr, addr->addr_len);
+			}
+			else {
+				if (rcpts != NULL) {
+					for (i = 0; i < rcpts->len; i++) {
+						addr = g_ptr_array_index(rcpts, i);
+
+						if (addr && addr->addr &&
+							match_smtp_data(task, arg,
+											addr->addr, addr->addr_len)) {
+							return TRUE;
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return FALSE;
+}
+
+static inline gboolean
+rspamd_check_ct_attr(const gchar *begin, gsize len,
+					 struct expression_argument *arg_pattern)
+{
+	rspamd_regexp_t *re;
+	gboolean r = FALSE;
+
+	if (arg_pattern->type == EXPRESSION_ARGUMENT_REGEXP) {
+		re = arg_pattern->data;
+
+		if (len > 0) {
+			r = rspamd_regexp_search(re,
+									 begin, len,
+									 NULL, NULL, FALSE, NULL);
+		}
+
+		if (r) {
+			return TRUE;
+		}
+	}
+	else {
+		/* Just do strcasecmp */
+		gsize plen = strlen(arg_pattern->data);
+
+		if (plen == len &&
+			g_ascii_strncasecmp(arg_pattern->data, begin, len) == 0) {
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+static gboolean
+rspamd_content_type_compare_param(struct rspamd_task *task,
+								  GArray *args,
+								  void *unused)
+{
+
+	struct expression_argument *arg, *arg1, *arg_pattern;
+	gboolean recursive = FALSE;
+	struct rspamd_mime_part *cur_part;
+	guint i;
+	rspamd_ftok_t srch;
+	struct rspamd_content_type_param *found = NULL, *cur;
+	const gchar *param_name;
+
+	if (args == NULL || args->len < 2) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+	g_assert(arg->type == EXPRESSION_ARGUMENT_NORMAL);
+	param_name = arg->data;
+	arg_pattern = &g_array_index(args, struct expression_argument, 1);
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, cur_part)
+	{
+		if (args->len >= 3) {
+			arg1 = &g_array_index(args, struct expression_argument, 2);
+			if (g_ascii_strncasecmp(arg1->data, "true",
+									sizeof("true") - 1) == 0) {
+				recursive = TRUE;
+			}
+		}
+		else {
+			/*
+			 * If user did not specify argument, let's assume that he wants
+			 * recursive search if mime part is multipart/mixed
+			 */
+			if (IS_PART_MULTIPART(cur_part)) {
+				recursive = TRUE;
+			}
+		}
+
+		rspamd_ftok_t lit;
+		RSPAMD_FTOK_FROM_STR(&srch, param_name);
+		RSPAMD_FTOK_FROM_STR(&lit, "charset");
+
+		if (rspamd_ftok_equal(&srch, &lit)) {
+			if (rspamd_check_ct_attr(cur_part->ct->charset.begin,
+									 cur_part->ct->charset.len, arg_pattern)) {
+				return TRUE;
+			}
+		}
+
+		RSPAMD_FTOK_FROM_STR(&lit, "boundary");
+		if (rspamd_ftok_equal(&srch, &lit)) {
+			if (rspamd_check_ct_attr(cur_part->ct->orig_boundary.begin,
+									 cur_part->ct->orig_boundary.len, arg_pattern)) {
+				return TRUE;
+			}
+		}
+
+		if (cur_part->ct->attrs) {
+			found = g_hash_table_lookup(cur_part->ct->attrs, &srch);
+
+			if (found) {
+				DL_FOREACH(found, cur)
+				{
+					if (rspamd_check_ct_attr(cur->value.begin,
+											 cur->value.len, arg_pattern)) {
+						return TRUE;
+					}
+				}
+			}
+		}
+
+		if (!recursive) {
+			break;
+		}
+	}
+
+	return FALSE;
+}
+
+static gboolean
+rspamd_content_type_has_param(struct rspamd_task *task,
+							  GArray *args,
+							  void *unused)
+{
+	struct expression_argument *arg, *arg1;
+	gboolean recursive = FALSE;
+	struct rspamd_mime_part *cur_part;
+	guint i;
+	rspamd_ftok_t srch;
+	struct rspamd_content_type_param *found = NULL;
+	const gchar *param_name;
+
+	if (args == NULL || args->len < 1) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	arg = &g_array_index(args, struct expression_argument, 0);
+	g_assert(arg->type == EXPRESSION_ARGUMENT_NORMAL);
+	param_name = arg->data;
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, cur_part)
+	{
+		if (args->len >= 2) {
+			arg1 = &g_array_index(args, struct expression_argument, 1);
+			if (g_ascii_strncasecmp(arg1->data, "true",
+									sizeof("true") - 1) == 0) {
+				recursive = TRUE;
+			}
+		}
+		else {
+			/*
+			 * If user did not specify argument, let's assume that he wants
+			 * recursive search if mime part is multipart/mixed
+			 */
+			if (IS_PART_MULTIPART(cur_part)) {
+				recursive = TRUE;
+			}
+		}
+
+
+		rspamd_ftok_t lit;
+		RSPAMD_FTOK_FROM_STR(&srch, param_name);
+		RSPAMD_FTOK_FROM_STR(&lit, "charset");
+
+		if (rspamd_ftok_equal(&srch, &lit)) {
+			if (cur_part->ct->charset.len > 0) {
+				return TRUE;
+			}
+		}
+
+		RSPAMD_FTOK_FROM_STR(&lit, "boundary");
+		if (rspamd_ftok_equal(&srch, &lit)) {
+			if (cur_part->ct->boundary.len > 0) {
+				return TRUE;
+			}
+		}
+
+		if (cur_part->ct->attrs) {
+			found = g_hash_table_lookup(cur_part->ct->attrs, &srch);
+
+			if (found) {
+				return TRUE;
+			}
+		}
+
+		if (!recursive) {
+			break;
+		}
+	}
+
+	return FALSE;
+}
+
+static gboolean
+rspamd_content_type_check(struct rspamd_task *task,
+						  GArray *args,
+						  gboolean check_subtype)
+{
+	rspamd_ftok_t *param_data, srch;
+	rspamd_regexp_t *re;
+	struct expression_argument *arg1, *arg_pattern;
+	struct rspamd_content_type *ct;
+	gint r = 0;
+	guint i;
+	gboolean recursive = FALSE;
+	struct rspamd_mime_part *cur_part;
+
+	if (args == NULL || args->len < 1) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	arg_pattern = &g_array_index(args, struct expression_argument, 0);
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, cur_part)
+	{
+		ct = cur_part->ct;
+
+		if (args->len >= 2) {
+			arg1 = &g_array_index(args, struct expression_argument, 1);
+			if (g_ascii_strncasecmp(arg1->data, "true",
+									sizeof("true") - 1) == 0) {
+				recursive = TRUE;
+			}
+		}
+		else {
+			/*
+			 * If user did not specify argument, let's assume that he wants
+			 * recursive search if mime part is multipart/mixed
+			 */
+			if (IS_PART_MULTIPART(cur_part)) {
+				recursive = TRUE;
+			}
+		}
+
+		if (check_subtype) {
+			param_data = &ct->subtype;
+		}
+		else {
+			param_data = &ct->type;
+		}
+
+		if (arg_pattern->type == EXPRESSION_ARGUMENT_REGEXP) {
+			re = arg_pattern->data;
+
+			if (param_data->len > 0) {
+				r = rspamd_regexp_search(re, param_data->begin, param_data->len,
+										 NULL, NULL, FALSE, NULL);
+			}
+
+			if (r) {
+				return TRUE;
+			}
+		}
+		else {
+			/* Just do strcasecmp */
+			srch.begin = arg_pattern->data;
+			srch.len = strlen(arg_pattern->data);
+
+			if (rspamd_ftok_casecmp(param_data, &srch) == 0) {
+				return TRUE;
+			}
+		}
+
+		/* Get next part */
+		if (!recursive) {
+			break;
+		}
+	}
+
+	return FALSE;
+}
+
+static gboolean
+rspamd_content_type_is_type(struct rspamd_task *task,
+							GArray *args,
+							void *unused)
+{
+	return rspamd_content_type_check(task, args, FALSE);
+}
+
+static gboolean
+rspamd_content_type_is_subtype(struct rspamd_task *task,
+							   GArray *args,
+							   void *unused)
+{
+	return rspamd_content_type_check(task, args, TRUE);
+}
+
+static gboolean
+compare_subtype(struct rspamd_task *task, struct rspamd_content_type *ct,
+				struct expression_argument *subtype)
+{
+	rspamd_regexp_t *re;
+	rspamd_ftok_t srch;
+	gint r = 0;
+
+	if (subtype == NULL || ct == NULL) {
+		msg_warn_task("invalid parameters passed");
+		return FALSE;
+	}
+	if (subtype->type == EXPRESSION_ARGUMENT_REGEXP) {
+		re = subtype->data;
+
+		if (ct->subtype.len > 0) {
+			r = rspamd_regexp_search(re, ct->subtype.begin, ct->subtype.len,
+									 NULL, NULL, FALSE, NULL);
+		}
+	}
+	else {
+		srch.begin = subtype->data;
+		srch.len = strlen(subtype->data);
+
+		/* Just do strcasecmp */
+		if (rspamd_ftok_casecmp(&ct->subtype, &srch) == 0) {
+			return TRUE;
+		}
+	}
+
+	return r;
+}
+
+static gboolean
+compare_len(struct rspamd_mime_part *part, guint min, guint max)
+{
+	if (min == 0 && max == 0) {
+		return TRUE;
+	}
+
+	if (min == 0) {
+		return part->parsed_data.len <= max;
+	}
+	else if (max == 0) {
+		return part->parsed_data.len >= min;
+	}
+	else {
+		return part->parsed_data.len >= min && part->parsed_data.len <= max;
+	}
+}
+
+static gboolean
+common_has_content_part(struct rspamd_task *task,
+						struct expression_argument *param_type,
+						struct expression_argument *param_subtype,
+						gint min_len,
+						gint max_len)
+{
+	rspamd_regexp_t *re;
+	struct rspamd_mime_part *part;
+	struct rspamd_content_type *ct;
+	rspamd_ftok_t srch;
+	gint r = 0;
+	guint i;
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+	{
+		ct = part->ct;
+
+		if (ct == NULL) {
+			continue;
+		}
+
+		if (param_type->type == EXPRESSION_ARGUMENT_REGEXP) {
+			re = param_type->data;
+
+			if (ct->type.len > 0) {
+				r = rspamd_regexp_search(re, ct->type.begin, ct->type.len,
+										 NULL, NULL, FALSE, NULL);
+			}
+
+			/* Also check subtype and length of the part */
+			if (r && param_subtype) {
+				r = compare_len(part, min_len, max_len) &&
+					compare_subtype(task, ct, param_subtype);
+
+				return r;
+			}
+		}
+		else {
+			/* Just do strcasecmp */
+			srch.begin = param_type->data;
+			srch.len = strlen(param_type->data);
+
+			if (rspamd_ftok_casecmp(&ct->type, &srch) == 0) {
+				if (param_subtype) {
+					if (compare_subtype(task, ct, param_subtype)) {
+						if (compare_len(part, min_len, max_len)) {
+							return TRUE;
+						}
+					}
+				}
+				else {
+					if (compare_len(part, min_len, max_len)) {
+						return TRUE;
+					}
+				}
+			}
+		}
+	}
+
+	return FALSE;
+}
+
+static gboolean
+rspamd_has_content_part(struct rspamd_task *task, GArray *args, void *unused)
+{
+	struct expression_argument *param_type = NULL, *param_subtype = NULL;
+
+	if (args == NULL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	param_type = &g_array_index(args, struct expression_argument, 0);
+	if (args->len >= 2) {
+		param_subtype = &g_array_index(args, struct expression_argument, 1);
+	}
+
+	return common_has_content_part(task, param_type, param_subtype, 0, 0);
+}
+
+static gboolean
+rspamd_has_content_part_len(struct rspamd_task *task,
+							GArray *args,
+							void *unused)
+{
+	struct expression_argument *param_type = NULL, *param_subtype = NULL;
+	gint min = 0, max = 0;
+	struct expression_argument *arg;
+
+	if (args == NULL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	param_type = &g_array_index(args, struct expression_argument, 0);
+
+	if (args->len >= 2) {
+		param_subtype = &g_array_index(args, struct expression_argument, 1);
+
+		if (args->len >= 3) {
+			arg = &g_array_index(args, struct expression_argument, 2);
+			errno = 0;
+			min = strtoul(arg->data, NULL, 10);
+			g_assert(arg->type == EXPRESSION_ARGUMENT_NORMAL);
+
+			if (errno != 0) {
+				msg_warn_task("invalid numeric value '%s': %s",
+							  (gchar *) arg->data,
+							  strerror(errno));
+				return FALSE;
+			}
+
+			if (args->len >= 4) {
+				arg = &g_array_index(args, struct expression_argument, 3);
+				g_assert(arg->type == EXPRESSION_ARGUMENT_NORMAL);
+				max = strtoul(arg->data, NULL, 10);
+
+				if (errno != 0) {
+					msg_warn_task("invalid numeric value '%s': %s",
+								  (gchar *) arg->data,
+								  strerror(errno));
+					return FALSE;
+				}
+			}
+		}
+	}
+
+	return common_has_content_part(task, param_type, param_subtype, min, max);
+}
+
+static gboolean
+rspamd_is_empty_body(struct rspamd_task *task,
+					 GArray *args,
+					 void *unused)
+{
+	struct rspamd_mime_part *part;
+	guint i;
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, parts), i, part)
+	{
+		if (part->parsed_data.len > 0) {
+			return FALSE;
+		}
+	}
+
+	return TRUE;
+}
+
+#define TASK_FLAG_READ(flag)               \
+	do {                                   \
+		result = !!(task->flags & (flag)); \
+	} while (0)
+
+#define TASK_GET_FLAG(flag, strname, macro)           \
+	do {                                              \
+		if (!found && strcmp((flag), strname) == 0) { \
+			TASK_FLAG_READ((macro));                  \
+			found = TRUE;                             \
+		}                                             \
+	} while (0)
+
+#define TASK_PROTOCOL_FLAG_READ(flag)               \
+	do {                                            \
+		result = !!(task->protocol_flags & (flag)); \
+	} while (0)
+
+#define TASK_GET_PROTOCOL_FLAG(flag, strname, macro)  \
+	do {                                              \
+		if (!found && strcmp((flag), strname) == 0) { \
+			TASK_PROTOCOL_FLAG_READ((macro));         \
+			found = TRUE;                             \
+		}                                             \
+	} while (0)
+
+
+static gboolean
+rspamd_has_flag_expr(struct rspamd_task *task,
+					 GArray *args,
+					 void *unused)
+{
+	gboolean found = FALSE, result = FALSE;
+	struct expression_argument *flag_arg;
+	const gchar *flag_str;
+
+	if (args == NULL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	flag_arg = &g_array_index(args, struct expression_argument, 0);
+
+	if (flag_arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("invalid parameter to function");
+		return FALSE;
+	}
+
+	flag_str = (const gchar *) flag_arg->data;
+
+	TASK_GET_FLAG(flag_str, "pass_all", RSPAMD_TASK_FLAG_PASS_ALL);
+	TASK_GET_FLAG(flag_str, "no_log", RSPAMD_TASK_FLAG_NO_LOG);
+	TASK_GET_FLAG(flag_str, "no_stat", RSPAMD_TASK_FLAG_NO_STAT);
+	TASK_GET_FLAG(flag_str, "skip", RSPAMD_TASK_FLAG_SKIP);
+	TASK_GET_PROTOCOL_FLAG(flag_str, "extended_urls",
+						   RSPAMD_TASK_PROTOCOL_FLAG_EXT_URLS);
+	TASK_GET_FLAG(flag_str, "learn_spam", RSPAMD_TASK_FLAG_LEARN_SPAM);
+	TASK_GET_FLAG(flag_str, "learn_ham", RSPAMD_TASK_FLAG_LEARN_HAM);
+	TASK_GET_FLAG(flag_str, "greylisted", RSPAMD_TASK_FLAG_GREYLISTED);
+	TASK_GET_FLAG(flag_str, "broken_headers",
+				  RSPAMD_TASK_FLAG_BROKEN_HEADERS);
+	TASK_GET_FLAG(flag_str, "skip_process",
+				  RSPAMD_TASK_FLAG_SKIP_PROCESS);
+	TASK_GET_PROTOCOL_FLAG(flag_str, "milter",
+						   RSPAMD_TASK_PROTOCOL_FLAG_MILTER);
+	TASK_GET_FLAG(flag_str, "bad_unicode",
+				  RSPAMD_TASK_FLAG_BAD_UNICODE);
+
+	if (!found) {
+		msg_warn_task("invalid flag name %s", flag_str);
+		return FALSE;
+	}
+
+	return result;
+}
+
+static gboolean
+rspamd_has_symbol_expr(struct rspamd_task *task,
+					   GArray *args,
+					   void *unused)
+{
+	struct expression_argument *sym_arg;
+	const gchar *symbol_str;
+
+	if (args == NULL) {
+		msg_warn_task("no parameters to function");
+		return FALSE;
+	}
+
+	sym_arg = &g_array_index(args, struct expression_argument, 0);
+
+	if (sym_arg->type != EXPRESSION_ARGUMENT_NORMAL) {
+		msg_warn_task("invalid parameter to function");
+		return FALSE;
+	}
+
+	symbol_str = (const gchar *) sym_arg->data;
+
+	if (rspamd_task_find_symbol_result(task, symbol_str, NULL)) {
+		return TRUE;
+	}
+
+	return FALSE;
+}
diff --git a/src/libmime/mime_expressions.h b/src/libmime/mime_expressions.h
new file mode 100644
index 0000000..a2ea3fe
--- /dev/null
+++ b/src/libmime/mime_expressions.h
@@ -0,0 +1,65 @@
+/**
+ * @file expressions.h
+ * Rspamd expressions API
+ */
+
+#ifndef RSPAMD_EXPRESSIONS_H
+#define RSPAMD_EXPRESSIONS_H
+
+#include "config.h"
+#include "expression.h"
+#include "contrib/libucl/ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_config;
+
+struct rspamd_mime_expr_ud {
+	struct rspamd_config *cfg;
+	const ucl_object_t *conf_obj;
+};
+
+extern const struct rspamd_atom_subr mime_expr_subr;
+
+/**
+ * Function's argument
+ */
+enum rspamd_expression_type {
+	EXPRESSION_ARGUMENT_NORMAL = 0,
+	EXPRESSION_ARGUMENT_BOOL,
+	EXPRESSION_ARGUMENT_REGEXP
+};
+struct expression_argument {
+	enum rspamd_expression_type type; /**< type of argument (text or other function)		*/
+	void *data;                       /**< pointer to its data							*/
+};
+
+
+typedef gboolean (*rspamd_internal_func_t)(struct rspamd_task *,
+										   GArray *args, void *user_data);
+
+
+/**
+ * Register specified function to rspamd internal functions list
+ * @param name name of function
+ * @param func pointer to function
+ */
+void register_expression_function(const gchar *name,
+								  rspamd_internal_func_t func,
+								  void *user_data);
+
+/**
+ * Set global limit of regexp data size to be processed
+ * @param limit new limit in bytes
+ * @return old limit value
+ */
+guint rspamd_mime_expression_set_re_limit(guint limit);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libmime/mime_headers.c b/src/libmime/mime_headers.c
new file mode 100644
index 0000000..2bd559d
--- /dev/null
+++ b/src/libmime/mime_headers.c
@@ -0,0 +1,1441 @@
+/*
+ * Copyright 2024 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "mime_headers.h"
+#include "smtp_parsers.h"
+#include "mime_encoding.h"
+#include "received.h"
+#include "contrib/uthash/utlist.h"
+#include "libserver/mempool_vars_internal.h"
+#include "libserver/cfg_file.h"
+#include "libutil/util.h"
+#include <unicode/utf8.h>
+
+KHASH_INIT(rspamd_mime_headers_htb, gchar *,
+		   struct rspamd_mime_header *, 1,
+		   rspamd_strcase_hash, rspamd_strcase_equal);
+
+struct rspamd_mime_headers_table {
+	khash_t(rspamd_mime_headers_htb) htb;
+	ref_entry_t ref;
+};
+
+static void
+rspamd_mime_header_check_special(struct rspamd_task *task,
+								 struct rspamd_mime_header *rh)
+{
+	guint64 h;
+	const gchar *p, *end;
+	gchar *id;
+	gint max_recipients = -1, len;
+
+	if (task->cfg) {
+		max_recipients = task->cfg->max_recipients;
+	}
+
+	h = rspamd_icase_hash(rh->name, strlen(rh->name), 0xdeadbabe);
+
+	switch (h) {
+	case 0x88705DC4D9D61ABULL: /* received */
+		if (rspamd_received_header_parse(task, rh->decoded, strlen(rh->decoded), rh)) {
+			rh->flags |= RSPAMD_HEADER_RECEIVED;
+		}
+		break;
+	case 0x76F31A09F4352521ULL: /* to */
+		MESSAGE_FIELD(task, rcpt_mime) = rspamd_email_address_from_mime(task->task_pool,
+																		rh->value, strlen(rh->value),
+																		MESSAGE_FIELD(task, rcpt_mime), max_recipients);
+		rh->flags |= RSPAMD_HEADER_TO | RSPAMD_HEADER_RCPT | RSPAMD_HEADER_UNIQUE;
+		break;
+	case 0x7EB117C1480B76ULL: /* cc */
+		MESSAGE_FIELD(task, rcpt_mime) = rspamd_email_address_from_mime(task->task_pool,
+																		rh->value, strlen(rh->value),
+																		MESSAGE_FIELD(task, rcpt_mime), max_recipients);
+		rh->flags |= RSPAMD_HEADER_CC | RSPAMD_HEADER_RCPT | RSPAMD_HEADER_UNIQUE;
+		break;
+	case 0xE4923E11C4989C8DULL: /* bcc */
+		MESSAGE_FIELD(task, rcpt_mime) = rspamd_email_address_from_mime(task->task_pool,
+																		rh->value, strlen(rh->value),
+																		MESSAGE_FIELD(task, rcpt_mime), max_recipients);
+		rh->flags |= RSPAMD_HEADER_BCC | RSPAMD_HEADER_RCPT | RSPAMD_HEADER_UNIQUE;
+		break;
+	case 0x41E1985EDC1CBDE4ULL: /* from */
+		MESSAGE_FIELD(task, from_mime) = rspamd_email_address_from_mime(task->task_pool,
+																		rh->value, strlen(rh->value),
+																		MESSAGE_FIELD(task, from_mime), max_recipients);
+		rh->flags |= RSPAMD_HEADER_FROM | RSPAMD_HEADER_SENDER | RSPAMD_HEADER_UNIQUE;
+		break;
+	case 0x43A558FC7C240226ULL: /* message-id */ {
+
+		rh->flags = RSPAMD_HEADER_MESSAGE_ID | RSPAMD_HEADER_UNIQUE;
+		p = rh->decoded;
+		len = rspamd_strip_smtp_comments_inplace(rh->decoded, strlen(p));
+		rh->decoded[len] = '\0'; /* Zero terminate after stripping */
+		/* Strip surrounding spaces */
+		rh->decoded = g_strstrip(rh->decoded);
+		end = p + len;
+
+		if (*p == '<') {
+			p++;
+		}
+
+		if (end > p) {
+			gchar *d;
+
+			if (*(end - 1) == '>') {
+				end--;
+			}
+
+			id = rspamd_mempool_alloc(task->task_pool, end - p + 1);
+			d = id;
+
+			while (p < end) {
+				if (g_ascii_isgraph(*p)) {
+					*d++ = *p++;
+				}
+				else {
+					*d++ = '?';
+					p++;
+				}
+			}
+
+			*d = '\0';
+
+			MESSAGE_FIELD(task, message_id) = id;
+		}
+
+		break;
+	}
+	case 0xB91D3910358E8212ULL: /* subject */
+		if (MESSAGE_FIELD(task, subject) == NULL) {
+			MESSAGE_FIELD(task, subject) = rh->decoded;
+		}
+		rh->flags = RSPAMD_HEADER_SUBJECT | RSPAMD_HEADER_UNIQUE;
+		break;
+	case 0xEE4AA2EAAC61D6F4ULL: /* return-path */
+		if (task->from_envelope == NULL) {
+			task->from_envelope = rspamd_email_address_from_smtp(rh->decoded,
+																 strlen(rh->decoded));
+		}
+		rh->flags = RSPAMD_HEADER_RETURN_PATH | RSPAMD_HEADER_UNIQUE;
+		break;
+	case 0xB9EEFAD2E93C2161ULL: /* delivered-to */
+		if (task->deliver_to == NULL) {
+			task->deliver_to = rh->decoded;
+		}
+		rh->flags = RSPAMD_HEADER_DELIVERED_TO;
+		break;
+	case 0x2EC3BFF3C393FC10ULL: /* date */
+	case 0xAC0DDB1A1D214CAULL:  /* sender */
+	case 0x54094572367AB695ULL: /* in-reply-to */
+	case 0x81CD9E9131AB6A9AULL: /* content-type */
+	case 0xC39BD9A75AA25B60ULL: /* content-transfer-encoding */
+	case 0xB3F6704CB3AD6589ULL: /* references */
+		rh->flags = RSPAMD_HEADER_UNIQUE;
+		break;
+	}
+}
+
+static void
+rspamd_mime_header_add(struct rspamd_task *task,
+					   khash_t(rspamd_mime_headers_htb) * target,
+					   struct rspamd_mime_header **order_ptr,
+					   struct rspamd_mime_header *rh,
+					   gboolean check_special)
+{
+	khiter_t k;
+	struct rspamd_mime_header *ex;
+	int res;
+
+	k = kh_put(rspamd_mime_headers_htb, target, rh->name, &res);
+
+	if (res == 0) {
+		ex = kh_value(target, k);
+		DL_APPEND(ex, rh);
+		msg_debug_task("append raw header %s: %s", rh->name, rh->value);
+	}
+	else {
+		kh_value(target, k) = rh;
+		rh->prev = rh;
+		rh->next = NULL;
+		msg_debug_task("add new raw header %s: %s", rh->name, rh->value);
+	}
+
+	LL_PREPEND2(*order_ptr, rh, ord_next);
+
+	if (check_special) {
+		rspamd_mime_header_check_special(task, rh);
+	}
+}
+
+
+/* Convert raw headers to a list of struct raw_header * */
+void rspamd_mime_headers_process(struct rspamd_task *task,
+								 struct rspamd_mime_headers_table *target,
+								 struct rspamd_mime_header **order_ptr,
+								 const gchar *in, gsize len,
+								 gboolean check_newlines)
+{
+	struct rspamd_mime_header *nh = NULL;
+	const gchar *p, *c, *end;
+	gchar *tmp, *tp;
+	gint state = 0, l, next_state = 100, err_state = 100, t_state;
+	gboolean valid_folding = FALSE, shift_by_one = FALSE;
+	guint nlines_count[RSPAMD_TASK_NEWLINES_MAX];
+	guint norder = 0;
+
+	p = in;
+	end = p + len;
+	c = p;
+	memset(nlines_count, 0, sizeof(nlines_count));
+	msg_debug_task("start processing headers");
+
+	while (p < end) {
+		/* FSM for processing headers */
+		switch (state) {
+		case 0:
+			/* Begin processing headers */
+			if (!g_ascii_isalpha(*p)) {
+				/* We have some garbage at the beginning of headers, skip this line */
+				state = 100;
+				next_state = 0;
+			}
+			else {
+				state = 1;
+				c = p;
+			}
+			break;
+		case 1:
+			/* We got something like header's name */
+			if (*p == ':') {
+				nh = rspamd_mempool_alloc0(task->task_pool,
+										   sizeof(struct rspamd_mime_header));
+				l = p - c;
+				tmp = rspamd_mempool_alloc(task->task_pool, l + 1);
+				rspamd_null_safe_copy(c, l, tmp, l + 1);
+				nh->name = tmp;
+				nh->flags |= RSPAMD_HEADER_EMPTY_SEPARATOR;
+				nh->raw_value = c;
+				nh->raw_len = p - c; /* Including trailing ':' */
+				p++;
+				state = 2;
+				c = p;
+			}
+			else if (g_ascii_isspace(*p)) {
+				/* Not header but some garbage */
+				if (target == MESSAGE_FIELD(task, raw_headers)) {
+					/* Do not propagate flag from the attachments */
+					task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+				}
+				state = 100;
+				next_state = 0;
+			}
+			else {
+				p++;
+			}
+			break;
+		case 2:
+			/* We got header's name, so skip any \t or spaces */
+			if (*p == '\t') {
+				nh->flags &= ~RSPAMD_HEADER_EMPTY_SEPARATOR;
+				nh->flags |= RSPAMD_HEADER_TAB_SEPARATED;
+				p++;
+			}
+			else if (*p == ' ') {
+				nh->flags &= ~RSPAMD_HEADER_EMPTY_SEPARATOR;
+				p++;
+			}
+			else if (*p == '\n' || *p == '\r') {
+
+				if (check_newlines) {
+					if (*p == '\n') {
+						nlines_count[RSPAMD_TASK_NEWLINES_LF]++;
+					}
+					else if (p + 1 < end && *(p + 1) == '\n') {
+						nlines_count[RSPAMD_TASK_NEWLINES_CRLF]++;
+					}
+					else {
+						nlines_count[RSPAMD_TASK_NEWLINES_CR]++;
+					}
+				}
+
+				/* Process folding */
+				state = 99;
+				l = p - c;
+				if (l > 0) {
+					tmp = rspamd_mempool_alloc(task->task_pool, l + 1);
+					rspamd_null_safe_copy(c, l, tmp, l + 1);
+					nh->separator = tmp;
+				}
+				next_state = 3;
+				err_state = 5;
+				c = p;
+			}
+			else {
+				/* Process value */
+				l = p - c;
+				if (l >= 0) {
+					tmp = rspamd_mempool_alloc(task->task_pool, l + 1);
+					rspamd_null_safe_copy(c, l, tmp, l + 1);
+					nh->separator = tmp;
+				}
+				c = p;
+				state = 3;
+			}
+			break;
+		case 3:
+			if (*p == '\r' || *p == '\n') {
+				/* Hold folding */
+				if (check_newlines) {
+					if (*p == '\n') {
+						nlines_count[RSPAMD_TASK_NEWLINES_LF]++;
+					}
+					else if (p + 1 < end && *(p + 1) == '\n') {
+						nlines_count[RSPAMD_TASK_NEWLINES_CRLF]++;
+					}
+					else {
+						nlines_count[RSPAMD_TASK_NEWLINES_CR]++;
+					}
+				}
+				state = 99;
+				next_state = 3;
+				err_state = 4;
+			}
+			else if (p + 1 == end) {
+				state = 4;
+			}
+			else {
+				p++;
+			}
+			break;
+		case 4:
+			/* Copy header's value */
+
+			/*
+			 * XXX:
+			 * The original decision to use here null terminated
+			 * strings was extremely poor!
+			 */
+			l = p - c;
+			tmp = rspamd_mempool_alloc(task->task_pool, l + 1);
+			tp = tmp;
+			t_state = 0;
+			while (l--) {
+				if (t_state == 0) {
+					/* Before folding */
+					if (*c == '\n' || *c == '\r') {
+						t_state = 1;
+						c++;
+						*tp++ = ' ';
+					}
+					else {
+						if (*c != '\0') {
+							*tp++ = *c++;
+						}
+						else {
+							c++;
+						}
+					}
+				}
+				else if (t_state == 1) {
+					/* Inside folding */
+					if (g_ascii_isspace(*c)) {
+						c++;
+					}
+					else {
+						t_state = 0;
+						if (*c != '\0') {
+							*tp++ = *c++;
+						}
+						else {
+							c++;
+						}
+					}
+				}
+			}
+			/* Strip last space that can be added by \r\n parsing */
+			if (tp > tmp && *(tp - 1) == ' ') {
+				tp--;
+			}
+
+			*tp = '\0';
+			/* Strip the initial spaces that could also be added by folding */
+			while (*tmp != '\0' && g_ascii_isspace(*tmp)) {
+				tmp++;
+			}
+
+			if (p + 1 == end) {
+				nh->raw_len = end - nh->raw_value;
+			}
+			else {
+				nh->raw_len = p - nh->raw_value;
+			}
+
+			nh->value = tmp;
+
+			gboolean broken_utf = FALSE;
+
+			nh->decoded = rspamd_mime_header_decode(task->task_pool,
+													nh->value, strlen(tmp), &broken_utf);
+
+			if (broken_utf) {
+				task->flags |= RSPAMD_TASK_FLAG_BAD_UNICODE;
+			}
+
+			if (nh->decoded == NULL) {
+				/* As we strip comments in place... */
+				nh->decoded = rspamd_mempool_strdup(task->task_pool, "");
+			}
+
+			/* We also validate utf8 and replace all non-valid utf8 chars */
+			rspamd_mime_charset_utf_enforce(nh->decoded, strlen(nh->decoded));
+			nh->order = norder++;
+			rspamd_mime_header_add(task, &target->htb, order_ptr, nh, check_newlines);
+			nh = NULL;
+			state = 0;
+			break;
+		case 5:
+			/* Header has only name, no value */
+			nh->value = rspamd_mempool_strdup(task->task_pool, "");
+			nh->decoded = rspamd_mempool_strdup(task->task_pool, "");
+			nh->raw_len = p - nh->raw_value;
+			if (shift_by_one) {
+				nh->raw_len++;
+			}
+			nh->order = norder++;
+			rspamd_mime_header_add(task, &target->htb, order_ptr, nh, check_newlines);
+			nh = NULL;
+			state = 0;
+			break;
+		case 99:
+			/* Folding state */
+			if (p + 1 == end) {
+				state = err_state;
+				/* Include the last character into the next header */
+				shift_by_one = TRUE;
+			}
+			else {
+				if (*p == '\r' || *p == '\n') {
+					p++;
+					valid_folding = FALSE;
+				}
+				else if (*p == '\t' || *p == ' ') {
+					/* Valid folding */
+					p++;
+					valid_folding = TRUE;
+				}
+				else {
+					if (valid_folding) {
+						debug_task("go to state: %d->%d", state, next_state);
+						state = next_state;
+					}
+					else {
+						/* Fall back */
+						debug_task("go to state: %d->%d", state, err_state);
+						state = err_state;
+					}
+				}
+			}
+			break;
+		case 100:
+			/* Fail state, skip line */
+
+			if (*p == '\r') {
+				if (p + 1 < end && *(p + 1) == '\n') {
+					nlines_count[RSPAMD_TASK_NEWLINES_CRLF]++;
+					p++;
+				}
+				p++;
+				state = next_state;
+			}
+			else if (*p == '\n') {
+				nlines_count[RSPAMD_TASK_NEWLINES_LF]++;
+
+				if (p + 1 < end && *(p + 1) == '\r') {
+					p++;
+				}
+				p++;
+				state = next_state;
+			}
+			else if (p + 1 == end) {
+				state = next_state;
+				p++;
+			}
+			else {
+				p++;
+			}
+			break;
+		}
+	}
+
+	/* Since we have prepended headers, we need to reverse the list to get the actual order */
+	LL_REVERSE(*order_ptr);
+
+	if (check_newlines) {
+		guint max_cnt = 0;
+		gint sel = 0;
+		rspamd_cryptobox_hash_state_t hs;
+		guchar hout[rspamd_cryptobox_HASHBYTES], *hexout;
+
+		for (gint i = RSPAMD_TASK_NEWLINES_CR; i < RSPAMD_TASK_NEWLINES_MAX; i++) {
+			if (nlines_count[i] > max_cnt) {
+				max_cnt = nlines_count[i];
+				sel = i;
+			}
+		}
+
+		MESSAGE_FIELD(task, nlines_type) = sel;
+
+		rspamd_cryptobox_hash_init(&hs, NULL, 0);
+
+		LL_FOREACH(*order_ptr, nh)
+		{
+			if (nh->name && nh->flags != RSPAMD_HEADER_RECEIVED) {
+				rspamd_cryptobox_hash_update(&hs, nh->name, strlen(nh->name));
+			}
+		}
+
+		rspamd_cryptobox_hash_final(&hs, hout);
+		hexout = rspamd_mempool_alloc(task->task_pool, sizeof(hout) * 2 + 1);
+		hexout[sizeof(hout) * 2] = '\0';
+		rspamd_encode_hex_buf(hout, sizeof(hout), hexout,
+							  sizeof(hout) * 2 + 1);
+		rspamd_mempool_set_variable(task->task_pool,
+									RSPAMD_MEMPOOL_HEADERS_HASH,
+									hexout, NULL);
+	}
+}
+
+static void
+rspamd_mime_header_maybe_save_token(rspamd_mempool_t *pool,
+									GString *out,
+									GByteArray *token,
+									GByteArray *decoded_token,
+									rspamd_ftok_t *old_charset,
+									rspamd_ftok_t *new_charset)
+{
+	if (new_charset->len == 0) {
+		g_assert_not_reached();
+	}
+
+	if (old_charset->len > 0) {
+		if (rspamd_ftok_casecmp(new_charset, old_charset) == 0) {
+			rspamd_ftok_t srch;
+
+			/*
+			 * Special case for iso-2022-jp:
+			 * https://github.com/vstakhov/rspamd/issues/1669
+			 */
+			RSPAMD_FTOK_ASSIGN(&srch, "iso-2022-jp");
+
+			if (rspamd_ftok_casecmp(new_charset, &srch) != 0) {
+				/* We can concatenate buffers, just return */
+				return;
+			}
+		}
+	}
+
+	/* We need to flush and decode old token to out string */
+	if (rspamd_mime_to_utf8_byte_array(token, decoded_token, pool,
+									   rspamd_mime_detect_charset(new_charset, pool))) {
+		g_string_append_len(out, decoded_token->data, decoded_token->len);
+	}
+
+	/* We also reset buffer */
+	g_byte_array_set_size(token, 0);
+	/*
+	 * Propagate charset
+	 *
+	 * Here are dragons: we save the original charset to allow buffers concat
+	 * in the condition at the beginning of the function.
+	 * However, it will likely cause unnecessary calls for
+	 * `rspamd_mime_detect_charset` which could be relatively expensive.
+	 * But we ignore that for now...
+	 */
+	memcpy(old_charset, new_charset, sizeof(*old_charset));
+}
+
+static void
+rspamd_mime_header_sanity_check(GString *str)
+{
+	gsize i;
+	gchar t;
+
+	for (i = 0; i < str->len; i++) {
+		t = str->str[i];
+		if (!((t & 0x80) || g_ascii_isgraph(t))) {
+			if (g_ascii_isspace(t)) {
+				/* Replace spaces characters with plain space */
+				str->str[i] = ' ';
+			}
+			else {
+				str->str[i] = '?';
+			}
+		}
+	}
+}
+
+gchar *
+rspamd_mime_header_decode(rspamd_mempool_t *pool, const gchar *in,
+						  gsize inlen, gboolean *invalid_utf)
+{
+	GString *out;
+	const guchar *c, *p, *end;
+	const gchar *tok_start = NULL;
+	gsize tok_len = 0, pos;
+	GByteArray *token = NULL, *decoded;
+	rspamd_ftok_t cur_charset = {0, NULL}, old_charset = {0, NULL};
+	gint encoding;
+	gssize r;
+	guint qmarks = 0;
+	gchar *ret;
+	enum {
+		parse_normal = 0,
+		got_eqsign,
+		got_encoded_start,
+		got_more_qmark,
+		skip_spaces,
+	} state = parse_normal;
+
+	g_assert(in != NULL);
+
+	c = in;
+	p = in;
+	end = in + inlen;
+	out = g_string_sized_new(inlen);
+	token = g_byte_array_sized_new(80);
+	decoded = g_byte_array_sized_new(122);
+
+	while (p < end) {
+		switch (state) {
+		case parse_normal:
+			if (*p == '=') {
+				g_string_append_len(out, c, p - c);
+				c = p;
+				state = got_eqsign;
+			}
+			else if (*p >= 128) {
+				gint off = 0;
+				UChar32 uc;
+				/* Unencoded character */
+				g_string_append_len(out, c, p - c);
+				/* Check if that's valid UTF8 */
+				U8_NEXT(p, off, end - p, uc);
+
+				if (uc <= 0) {
+					c = p + 1;
+					/* 0xFFFD in UTF8 */
+					g_string_append_len(out, "   ", 3);
+					off = 0;
+					U8_APPEND_UNSAFE(out->str + out->len - 3,
+									 off, 0xfffd);
+
+					if (invalid_utf) {
+						*invalid_utf = TRUE;
+					}
+				}
+				else {
+					c = p;
+					p = p + off;
+					continue; /* To avoid p ++ after this block */
+				}
+			}
+			p++;
+			break;
+		case got_eqsign:
+			if (*p == '?') {
+				state = got_encoded_start;
+				qmarks = 0;
+			}
+			else {
+				g_string_append_len(out, c, 1);
+				c = p;
+				state = parse_normal;
+				continue; /* Deal with == case */
+			}
+			p++;
+			break;
+		case got_encoded_start:
+			if (*p == '?') {
+				state = got_more_qmark;
+				qmarks++;
+
+				/* Skip multiple ? signs */
+				p++;
+				while (p < end && *p == '?') {
+					p++;
+				}
+
+				continue;
+			}
+			p++;
+			break;
+		case got_more_qmark:
+			if (*p == '=') {
+				if (qmarks < 3) {
+					state = got_encoded_start;
+				}
+				else {
+					/* Finished encoded boundary */
+					if (*c == '"') {
+						/* Quoted string, non-RFC conformant but used by retards */
+						c++;
+					}
+					if (rspamd_rfc2047_parser(c, p - c + 1, &encoding,
+											  &cur_charset.begin, &cur_charset.len,
+											  &tok_start, &tok_len)) {
+						/* We have a token, so we can decode it from `encoding` */
+						if (token->len > 0) {
+							if (old_charset.len == 0) {
+								memcpy(&old_charset, &cur_charset,
+									   sizeof(old_charset));
+							}
+
+							rspamd_mime_header_maybe_save_token(pool, out,
+																token, decoded,
+																&old_charset, &cur_charset);
+						}
+
+						qmarks = 0;
+						pos = token->len;
+						g_byte_array_set_size(token, pos + tok_len);
+
+						if (encoding == RSPAMD_RFC2047_QP) {
+							r = rspamd_decode_qp2047_buf(tok_start, tok_len,
+														 token->data + pos, tok_len);
+
+							if (r != -1) {
+								token->len = pos + r;
+							}
+							else {
+								/* Cannot decode qp */
+								token->len -= tok_len;
+							}
+						}
+						else {
+							if (rspamd_cryptobox_base64_decode(tok_start, tok_len,
+															   token->data + pos, &tok_len)) {
+								token->len = pos + tok_len;
+							}
+							else {
+								/* Cannot decode */
+								token->len -= tok_len;
+							}
+						}
+
+						c = p + 1;
+						state = skip_spaces;
+					}
+					else {
+						/* Not encoded-word */
+						old_charset.len = 0;
+
+						if (token->len > 0) {
+							rspamd_mime_header_maybe_save_token(pool, out,
+																token, decoded,
+																&old_charset, &cur_charset);
+						}
+
+						g_string_append_len(out, c, p - c);
+						c = p;
+						state = parse_normal;
+					}
+				} /* qmarks >= 3 */
+			}     /* p == '=' */
+			else {
+				state = got_encoded_start;
+			}
+			p++;
+			break;
+		case skip_spaces:
+			if (g_ascii_isspace(*p)) {
+				p++;
+			}
+			else if (*p == '=' && p < end - 1 && p[1] == '?') {
+				/* Next boundary, can glue */
+				c = p;
+				p += 2;
+				state = got_encoded_start;
+			}
+			else {
+				/* Need to save spaces and decoded token */
+				if (token->len > 0) {
+					old_charset.len = 0;
+					rspamd_mime_header_maybe_save_token(pool, out,
+														token, decoded,
+														&old_charset, &cur_charset);
+				}
+
+				g_string_append_len(out, c, p - c);
+				c = p;
+				state = parse_normal;
+			}
+			break;
+		}
+	}
+
+	/* Leftover */
+	switch (state) {
+	case skip_spaces:
+		if (token->len > 0 && cur_charset.len > 0) {
+			old_charset.len = 0;
+			rspamd_mime_header_maybe_save_token(pool, out,
+												token, decoded,
+												&old_charset, &cur_charset);
+		}
+		break;
+	default:
+		/* Just copy leftover */
+		if (p > c) {
+			g_string_append_len(out, c, p - c);
+		}
+		break;
+	}
+
+	g_byte_array_free(token, TRUE);
+	g_byte_array_free(decoded, TRUE);
+	rspamd_mime_header_sanity_check(out);
+	rspamd_mempool_notify_alloc(pool, out->len);
+	ret = g_string_free(out, FALSE);
+	rspamd_mempool_add_destructor(pool, g_free, ret);
+
+	return ret;
+}
+
+gchar *
+rspamd_mime_header_encode(const gchar *in, gsize len)
+{
+	const gchar *p = in, *end = in + len;
+	gchar *out, encode_buf[80 * sizeof(guint32)];
+	GString *res;
+	gboolean need_encoding = FALSE;
+
+	/* Check if we need to encode */
+	while (p < end) {
+		if ((((guchar) *p) & 0x80) != 0) {
+			need_encoding = TRUE;
+			break;
+		}
+		p++;
+	}
+
+	if (!need_encoding) {
+		out = g_malloc(len + 1);
+		rspamd_strlcpy(out, in, len + 1);
+	}
+	else {
+		/* Need encode */
+		gsize ulen, pos;
+		gint r;
+		const gchar *prev;
+		/* Choose step: =?UTF-8?Q?<qp>?= should be less than 76 chars */
+		guint step = (76 - 12) / 3 + 1;
+
+		ulen = g_utf8_strlen(in, len);
+		res = g_string_sized_new(len * 2 + 1);
+		pos = 0;
+		prev = in;
+		/* Adjust chunk size for unicode average length */
+		step *= 1.0 * ulen / (gdouble) len;
+
+		while (pos < ulen) {
+			p = g_utf8_offset_to_pointer(in, pos);
+
+			if (p > prev) {
+				/* Encode and print */
+				r = rspamd_encode_qp2047_buf(prev, p - prev,
+											 encode_buf, sizeof(encode_buf));
+
+				if (r != -1) {
+					if (res->len > 0) {
+						rspamd_printf_gstring(res, " =?UTF-8?Q?%*s?=", r,
+											  encode_buf);
+					}
+					else {
+						rspamd_printf_gstring(res, "=?UTF-8?Q?%*s?=", r,
+											  encode_buf);
+					}
+				}
+			}
+
+			pos += MIN(step, ulen - pos);
+			prev = p;
+		}
+
+		/* Leftover */
+		if (prev < end) {
+			r = rspamd_encode_qp2047_buf(prev, end - prev,
+										 encode_buf, sizeof(encode_buf));
+
+			if (r != -1) {
+				if (res->len > 0) {
+					rspamd_printf_gstring(res, " =?UTF-8?Q?%*s?=", r,
+										  encode_buf);
+				}
+				else {
+					rspamd_printf_gstring(res, "=?UTF-8?Q?%*s?=", r,
+										  encode_buf);
+				}
+			}
+		}
+
+		out = g_string_free(res, FALSE);
+	}
+
+	return out;
+}
+
+gchar *
+rspamd_mime_message_id_generate(const gchar *fqdn)
+{
+	GString *out;
+	guint64 rnd, clk;
+
+	out = g_string_sized_new(strlen(fqdn) + 22);
+	rnd = ottery_rand_uint64();
+	clk = rspamd_get_calendar_ticks() * 1e6;
+
+	rspamd_printf_gstring(out, "%*bs.%*bs@%s",
+						  (gint) sizeof(guint64) - 3, (guchar *) &clk,
+						  (gint) sizeof(guint64), (gchar *) &rnd,
+						  fqdn);
+
+	return g_string_free(out, FALSE);
+}
+
+struct rspamd_mime_header *
+rspamd_message_get_header_from_hash(struct rspamd_mime_headers_table *hdrs,
+									const gchar *field,
+									gboolean need_modified)
+{
+	if (hdrs == NULL) {
+		return NULL;
+	}
+
+	khiter_t k;
+	khash_t(rspamd_mime_headers_htb) *htb = &hdrs->htb;
+	struct rspamd_mime_header *hdr;
+
+	if (htb) {
+		k = kh_get(rspamd_mime_headers_htb, htb, (gchar *) field);
+
+		if (k == kh_end(htb)) {
+			return NULL;
+		}
+
+		hdr = kh_value(htb, k);
+
+		if (!need_modified) {
+			if (hdr->flags & RSPAMD_HEADER_NON_EXISTING) {
+				return NULL;
+			}
+
+			return hdr;
+		}
+		else {
+			if (hdr->flags & RSPAMD_HEADER_MODIFIED) {
+				return hdr->modified_chain;
+			}
+
+			return hdr;
+		}
+	}
+
+	return NULL;
+}
+
+struct rspamd_mime_header *
+rspamd_message_get_header_array(struct rspamd_task *task, const gchar *field,
+								gboolean need_modified)
+{
+	return rspamd_message_get_header_from_hash(
+		MESSAGE_FIELD_CHECK(task, raw_headers),
+		field, need_modified);
+}
+
+gsize rspamd_mime_headers_count(struct rspamd_mime_headers_table *hdrs)
+{
+	if (hdrs) {
+		return kh_size(&hdrs->htb);
+	}
+
+	return 0;
+}
+
+bool rspamd_mime_headers_foreach(const struct rspamd_mime_headers_table *hdrs,
+								 rspamd_hdr_traverse_func_t func, void *ud)
+{
+	const gchar *name;
+	struct rspamd_mime_header *hdr;
+
+	kh_foreach(&hdrs->htb, name, hdr, {
+		if (!func(name, hdr, ud)) {
+			return false;
+		}
+	});
+
+	return true;
+}
+
+static void
+rspamd_message_headers_dtor(struct rspamd_mime_headers_table *hdrs)
+{
+	if (hdrs) {
+		kfree(hdrs->htb.keys);
+		kfree(hdrs->htb.vals);
+		kfree(hdrs->htb.flags);
+		g_free(hdrs);
+	}
+}
+
+struct rspamd_mime_headers_table *
+rspamd_message_headers_ref(struct rspamd_mime_headers_table *hdrs)
+{
+	REF_RETAIN(hdrs);
+
+	return hdrs;
+}
+
+void rspamd_message_headers_unref(struct rspamd_mime_headers_table *hdrs)
+{
+	REF_RELEASE(hdrs);
+}
+
+struct rspamd_mime_headers_table *
+rspamd_message_headers_new(void)
+{
+	struct rspamd_mime_headers_table *nhdrs;
+
+	nhdrs = g_malloc0(sizeof(*nhdrs));
+	REF_INIT_RETAIN(nhdrs, rspamd_message_headers_dtor);
+
+	return nhdrs;
+}
+
+gsize rspamd_message_header_unfold_inplace(char *hdr, gsize len)
+{
+	/*
+	 * t - tortoise (destination)
+	 * h - hare (source)
+	 */
+	char *t = hdr, *h = hdr, *end = (hdr + len);
+	enum {
+		copy_chars,
+		folding_cr,
+		folding_lf,
+		folding_ws,
+	} state = copy_chars;
+
+	while (h < end) {
+		switch (state) {
+		case copy_chars:
+			if (*h == '\r') {
+				state = folding_cr;
+				h++;
+			}
+			else if (*h == '\n') {
+				state = folding_lf;
+				h++;
+			}
+			else {
+				*t++ = *h++;
+			}
+			break;
+		case folding_cr:
+			if (*h == '\n') {
+				state = folding_lf;
+				h++;
+			}
+			else if (g_ascii_isspace(*h)) {
+				state = folding_ws;
+				h++;
+			}
+			else {
+				/* It is weird, not like a folding, so we need to revert back */
+				*t++ = '\r';
+				state = copy_chars;
+			}
+			break;
+		case folding_lf:
+			if (g_ascii_isspace(*h)) {
+				state = folding_ws;
+				h++;
+			}
+			else {
+				/* It is weird, not like a folding, so we need to revert back */
+				*t++ = '\n';
+				state = copy_chars;
+			}
+			break;
+		case folding_ws:
+			if (!g_ascii_isspace(*h)) {
+				*t++ = ' ';
+				state = copy_chars;
+			}
+			else {
+				h++;
+			}
+			break;
+		}
+	}
+
+	return t - hdr;
+}
+
+void rspamd_message_set_modified_header(struct rspamd_task *task,
+										struct rspamd_mime_headers_table *hdrs,
+										const gchar *hdr_name,
+										const ucl_object_t *obj,
+										struct rspamd_mime_header **order_ptr)
+{
+	khiter_t k;
+	khash_t(rspamd_mime_headers_htb) *htb = &hdrs->htb;
+	struct rspamd_mime_header *hdr_elt, *existing_chain;
+	int i;
+
+	if (htb) {
+		k = kh_get(rspamd_mime_headers_htb, htb, (gchar *) hdr_name);
+
+		if (k == kh_end(htb)) {
+			hdr_elt = rspamd_mempool_alloc0(task->task_pool, sizeof(*hdr_elt));
+
+			hdr_elt->flags |= RSPAMD_HEADER_MODIFIED | RSPAMD_HEADER_NON_EXISTING;
+			hdr_elt->name = rspamd_mempool_strdup(task->task_pool, hdr_name);
+
+			int r;
+			k = kh_put(rspamd_mime_headers_htb, htb, hdr_elt->name, &r);
+
+			kh_value(htb, k) = hdr_elt;
+
+			if (order_ptr) {
+				/*
+				 * This iterates over all headers in O(N), but we have no other options here, as the
+				 * list is already set.
+				 */
+				LL_APPEND2(*order_ptr, hdr_elt, ord_next);
+			}
+		}
+		else {
+			hdr_elt = kh_value(htb, k);
+		}
+	}
+	else {
+		/* No hash, no modification */
+		msg_err_task("internal error: calling for set_modified_header for no headers");
+		return;
+	}
+
+	if (hdr_elt->flags & RSPAMD_HEADER_MODIFIED) {
+		existing_chain = hdr_elt->modified_chain;
+	}
+	else {
+		existing_chain = hdr_elt;
+	}
+
+	const ucl_object_t *elt, *cur;
+	ucl_object_iter_t it;
+
+	/* First, deal with removed headers, copying the relevant headers with remove flag */
+	elt = ucl_object_lookup(obj, "remove");
+
+	/*
+	 * remove:  {1, 2 ...}
+	 * where number is the header's position starting from '1'
+	 */
+	if (elt && ucl_object_type(elt) == UCL_ARRAY) {
+		/* First, use a temporary array to keep all headers */
+		GPtrArray *existing_ar = g_ptr_array_new();
+		struct rspamd_mime_header *cur_hdr;
+
+		/* Exclude removed headers */
+		LL_FOREACH(existing_chain, cur_hdr)
+		{
+			if (!(cur_hdr->flags & RSPAMD_HEADER_REMOVED)) {
+				g_ptr_array_add(existing_ar, cur_hdr);
+			}
+		}
+
+		it = NULL;
+
+		while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+			if (ucl_object_type(cur) == UCL_INT) {
+				int ord = ucl_object_toint(cur);
+
+				if (ord == 0) {
+					/* Remove all headers in the existing chain */
+					PTR_ARRAY_FOREACH(existing_ar, i, cur_hdr)
+					{
+						cur_hdr->flags |= RSPAMD_HEADER_MODIFIED | RSPAMD_HEADER_REMOVED;
+					}
+				}
+				else if (ord > 0) {
+					/* Start from the top */
+
+					if (ord <= existing_ar->len) {
+						cur_hdr = g_ptr_array_index(existing_ar, ord - 1);
+						cur_hdr->flags |= RSPAMD_HEADER_MODIFIED | RSPAMD_HEADER_REMOVED;
+					}
+				}
+				else {
+					/* Start from the bottom; ord < 0 */
+					if ((-ord) <= existing_ar->len) {
+						cur_hdr = g_ptr_array_index(existing_ar, existing_ar->len + ord);
+						cur_hdr->flags |= RSPAMD_HEADER_MODIFIED | RSPAMD_HEADER_REMOVED;
+					}
+				}
+			}
+		}
+
+		/*
+		 * Next, we return all headers modified to the existing chain
+		 * This implies an additional copy of all structures but is safe enough to
+		 * deal with it
+		 */
+		hdr_elt->flags |= RSPAMD_HEADER_MODIFIED;
+		hdr_elt->modified_chain = NULL;
+
+		PTR_ARRAY_FOREACH(existing_ar, i, cur_hdr)
+		{
+			if (!(cur_hdr->flags & RSPAMD_HEADER_REMOVED)) {
+				struct rspamd_mime_header *nhdr = rspamd_mempool_alloc(
+					task->task_pool, sizeof(*nhdr));
+				memcpy(nhdr, cur_hdr, sizeof(*nhdr));
+				nhdr->modified_chain = NULL;
+				nhdr->prev = NULL;
+				nhdr->next = NULL;
+				nhdr->ord_next = NULL;
+
+				DL_APPEND(hdr_elt->modified_chain, nhdr);
+			}
+		}
+
+		g_ptr_array_free(existing_ar, TRUE);
+
+		/* End of headers removal logic */
+	}
+
+	/* We can now deal with headers additions */
+	elt = ucl_object_lookup(obj, "add");
+	if (elt && ucl_object_type(elt) == UCL_ARRAY) {
+		if (!(hdr_elt->flags & RSPAMD_HEADER_MODIFIED)) {
+			/* Copy the header itself to the modified chain */
+			struct rspamd_mime_header *nhdr;
+			hdr_elt->flags |= RSPAMD_HEADER_MODIFIED;
+			nhdr = rspamd_mempool_alloc(
+				task->task_pool, sizeof(*nhdr));
+			memcpy(nhdr, hdr_elt, sizeof(*hdr_elt));
+			nhdr->modified_chain = NULL;
+			nhdr->next = NULL;
+			nhdr->ord_next = NULL;
+			nhdr->prev = nhdr;
+			hdr_elt->modified_chain = nhdr;
+		}
+
+		/*
+		 * add:  {{1, "foo"}, {-1, "bar"} ...}
+		 * where number is the header's position starting from '1'
+		 */
+		it = NULL;
+
+		while ((cur = ucl_object_iterate(elt, &it, true)) != NULL) {
+			if (ucl_object_type(cur) == UCL_ARRAY) {
+				const ucl_object_t *order = ucl_array_find_index(cur, 0),
+								   *value = ucl_array_find_index(cur, 1);
+
+				if (order && value &&
+					(ucl_object_type(order) == UCL_INT &&
+					 ucl_object_type(value) == UCL_STRING)) {
+					int ord = ucl_object_toint(order);
+					const char *raw_value;
+					gsize raw_len;
+
+					raw_value = ucl_object_tolstring(value, &raw_len);
+
+					if (raw_len == 0) {
+						continue;
+					}
+
+					struct rspamd_mime_header *nhdr = rspamd_mempool_alloc0(
+						task->task_pool, sizeof(*nhdr));
+
+					nhdr->flags |= RSPAMD_HEADER_ADDED;
+					nhdr->name = hdr_elt->name;
+					nhdr->value = rspamd_mempool_alloc(task->task_pool,
+													   raw_len + 1);
+					/* Strlcpy will ensure that value will have no embedded \0 */
+					rspamd_strlcpy(nhdr->value, raw_value, raw_len + 1);
+					gsize value_len = rspamd_message_header_unfold_inplace(nhdr->value, raw_len);
+					nhdr->value[value_len] = '\0';
+
+					/* Deal with the raw value */
+					size_t namelen = strlen(hdr_elt->name);
+					char *rawbuf = rspamd_mempool_alloc(task->task_pool, namelen +
+																			 raw_len +
+																			 sizeof(": \r\n"));
+					/* Name: value<newline> */
+					nhdr->raw_value = rawbuf;
+					memcpy(rawbuf, hdr_elt->name, namelen);
+					rawbuf += namelen;
+					memcpy(rawbuf, ": ", sizeof(": ") - 1);
+					nhdr->separator = rspamd_mempool_strdup(task->task_pool, " ");
+					rawbuf += sizeof(": ") - 1;
+					memcpy(rawbuf, raw_value, raw_len);
+					nhdr->raw_len = raw_len;
+
+					if (MESSAGE_FIELD(task, nlines_type) == RSPAMD_TASK_NEWLINES_LF) {
+						rawbuf[raw_len++] = '\n';
+					}
+					else {
+						rawbuf[raw_len++] = '\r';
+
+						if (MESSAGE_FIELD(task, nlines_type) == RSPAMD_TASK_NEWLINES_CRLF) {
+							rawbuf[raw_len++] = '\n';
+						}
+					}
+
+					rawbuf[raw_len] = '\0';
+
+					nhdr->decoded = rspamd_mime_header_decode(task->task_pool,
+															  raw_value, nhdr->raw_len,
+															  NULL);
+
+					/* Now find a position to insert a value */
+					struct rspamd_mime_header **pos = &hdr_elt->modified_chain;
+
+					if (ord == 0) {
+						DL_PREPEND(hdr_elt->modified_chain, nhdr);
+					}
+					else if (ord == -1) {
+						DL_APPEND(hdr_elt->modified_chain, nhdr);
+					}
+					else if (ord > 0) {
+						while (ord > 0 && (*pos)) {
+							ord--;
+							pos = &((*pos)->next);
+						}
+						if (*pos) {
+							/* pos is &(elt)->next */
+							nhdr->next = (*pos);
+							nhdr->prev = (*pos)->prev;
+							(*pos)->prev = nhdr;
+							*pos = nhdr;
+						}
+						else {
+							/* Last element */
+							DL_APPEND(*pos, nhdr);
+						}
+					}
+					else {
+						/* NYI: negative order is not defined */
+						msg_err_task("internal error: calling for set_modified_header "
+									 "with negative add order header");
+					}
+				}
+				else {
+					msg_err_task("internal error: calling for set_modified_header "
+								 "with invalid header");
+				}
+			}
+		}
+	}
+}
+
+gsize rspamd_strip_smtp_comments_inplace(gchar *input, gsize len)
+{
+	enum parser_state {
+		parse_normal,
+		parse_obrace,
+		parse_comment,
+		parse_quoted_copy,
+		parse_quoted_ignore,
+	} state = parse_normal,
+	  next_state = parse_normal;
+	gchar *d = input, *end = input + len, *start = input;
+	gchar t;
+	int obraces = 0, ebraces = 0;
+
+	while (input < end) {
+		t = *input;
+		switch (state) {
+		case parse_normal:
+			if (t == '(') {
+				state = parse_obrace;
+			}
+			else if (t == '\\') {
+				state = parse_quoted_copy;
+				next_state = parse_normal;
+			}
+			else {
+				*d++ = t;
+			}
+			input++;
+			break;
+		case parse_obrace:
+			obraces++;
+			if (t == '(') {
+				obraces++;
+			}
+			else if (t == ')') {
+				ebraces++;
+
+				if (obraces == ebraces) {
+					obraces = 0;
+					ebraces = 0;
+					state = parse_normal;
+				}
+			}
+			else if (t == '\\') {
+				state = parse_quoted_ignore;
+				next_state = parse_comment;
+			}
+			else {
+				state = parse_comment;
+			}
+			input++;
+			break;
+		case parse_comment:
+			if (t == '(') {
+				state = parse_obrace;
+			}
+			else if (t == ')') {
+				ebraces++;
+
+				if (obraces == ebraces) {
+					obraces = 0;
+					ebraces = 0;
+					state = parse_normal;
+				}
+			}
+			else if (t == '\\') {
+				state = parse_quoted_ignore;
+				next_state = parse_comment;
+			}
+			input++;
+			break;
+		case parse_quoted_copy:
+			*d++ = t;
+			state = next_state;
+			input++;
+			break;
+		case parse_quoted_ignore:
+			state = next_state;
+			input++;
+			break;
+		}
+	}
+
+	return (d - start);
+}
\ No newline at end of file
diff --git a/src/libmime/mime_headers.h b/src/libmime/mime_headers.h
new file mode 100644
index 0000000..60015a2
--- /dev/null
+++ b/src/libmime/mime_headers.h
@@ -0,0 +1,200 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_MIME_HEADERS_H_
+#define SRC_LIBMIME_MIME_HEADERS_H_
+
+#include "config.h"
+#include "libutil/mem_pool.h"
+#include "libutil/addr.h"
+#include "khash.h"
+#include "contrib/libucl/ucl.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+
+enum rspamd_rfc2047_encoding {
+	RSPAMD_RFC2047_QP = 0,
+	RSPAMD_RFC2047_BASE64,
+};
+
+enum rspamd_mime_header_flags {
+	RSPAMD_HEADER_GENERIC = 0u,
+	RSPAMD_HEADER_RECEIVED = 1u << 0u,
+	RSPAMD_HEADER_TO = 1u << 2u,
+	RSPAMD_HEADER_CC = 1u << 3u,
+	RSPAMD_HEADER_BCC = 1u << 4u,
+	RSPAMD_HEADER_FROM = 1u << 5u,
+	RSPAMD_HEADER_MESSAGE_ID = 1u << 6u,
+	RSPAMD_HEADER_SUBJECT = 1u << 7u,
+	RSPAMD_HEADER_RETURN_PATH = 1u << 8u,
+	RSPAMD_HEADER_DELIVERED_TO = 1u << 9u,
+	RSPAMD_HEADER_SENDER = 1u << 10u,
+	RSPAMD_HEADER_RCPT = 1u << 11u,
+	RSPAMD_HEADER_UNIQUE = 1u << 12u,
+	RSPAMD_HEADER_EMPTY_SEPARATOR = 1u << 13u,
+	RSPAMD_HEADER_TAB_SEPARATED = 1u << 14u,
+	RSPAMD_HEADER_MODIFIED = 1u << 15u,     /* Means we need to check modified chain */
+	RSPAMD_HEADER_ADDED = 1u << 16u,        /* A header has been artificially added */
+	RSPAMD_HEADER_REMOVED = 1u << 17u,      /* A header has been artificially removed */
+	RSPAMD_HEADER_NON_EXISTING = 1u << 18u, /* Header was not in the original message */
+};
+
+struct rspamd_mime_header {
+	const gchar *raw_value; /* As it is in the message (unfolded and unparsed) */
+	gsize raw_len;
+	guint order;
+	int flags; /* see enum rspamd_mime_header_flags */
+	/* These are zero terminated (historically) */
+	gchar *name; /* Also used for key */
+	gchar *value;
+	gchar *separator;
+	gchar *decoded;
+	struct rspamd_mime_header *modified_chain; /* Headers modified during transform */
+	struct rspamd_mime_header *prev, *next;    /* Headers with the same name */
+	struct rspamd_mime_header *ord_next;       /* Overall order of headers, slist */
+};
+
+struct rspamd_mime_headers_table;
+
+/**
+ * Process headers and store them in `target`
+ * @param task
+ * @param target
+ * @param in
+ * @param len
+ * @param check_newlines
+ */
+void rspamd_mime_headers_process(struct rspamd_task *task,
+								 struct rspamd_mime_headers_table *target,
+								 struct rspamd_mime_header **order_ptr,
+								 const gchar *in, gsize len,
+								 gboolean check_newlines);
+
+/**
+ * Perform rfc2047 decoding of a header
+ * @param pool
+ * @param in
+ * @param inlen
+ * @return
+ */
+gchar *rspamd_mime_header_decode(rspamd_mempool_t *pool, const gchar *in,
+								 gsize inlen, gboolean *invalid_utf);
+
+/**
+ * Encode mime header if needed
+ * @param in
+ * @param len
+ * @return newly allocated encoded header
+ */
+gchar *rspamd_mime_header_encode(const gchar *in, gsize len);
+
+/**
+ * Generate new unique message id
+ * @param fqdn
+ * @return
+ */
+gchar *rspamd_mime_message_id_generate(const gchar *fqdn);
+
+/**
+ * Get an array of header's values with specified header's name using raw headers
+ * @param task worker task structure
+ * @param field header's name
+ * @return An array of header's values or NULL. It is NOT permitted to free array or values.
+ */
+struct rspamd_mime_header *
+rspamd_message_get_header_array(struct rspamd_task *task,
+								const gchar *field,
+								gboolean need_modified);
+
+/**
+ * Get an array of header's values with specified header's name using raw headers
+ * @param htb hash table indexed by header name (caseless) with ptr arrays as elements
+ * @param field header's name
+ * @return An array of header's values or NULL. It is NOT permitted to free array or values.
+ */
+struct rspamd_mime_header *
+rspamd_message_get_header_from_hash(struct rspamd_mime_headers_table *hdrs,
+									const gchar *field,
+									gboolean need_modified);
+
+/**
+ * Modifies a header (or insert one if not found)
+ * @param hdrs
+ * @param hdr_name
+ * @param obj an array of modified values
+ *
+ */
+void rspamd_message_set_modified_header(struct rspamd_task *task,
+										struct rspamd_mime_headers_table *hdrs,
+										const gchar *hdr_name,
+										const ucl_object_t *obj,
+										struct rspamd_mime_header **order_ptr);
+
+/**
+ * Cleans up hash table of the headers
+ * @param htb
+ */
+void rspamd_message_headers_unref(struct rspamd_mime_headers_table *hdrs);
+
+struct rspamd_mime_headers_table *rspamd_message_headers_ref(struct rspamd_mime_headers_table *hdrs);
+
+/**
+ * Init headers hash
+ * @return
+ */
+struct rspamd_mime_headers_table *rspamd_message_headers_new(void);
+
+/**
+ * Returns size for a headers table
+ * @param hdrs
+ * @return
+ */
+gsize rspamd_mime_headers_count(struct rspamd_mime_headers_table *hdrs);
+
+typedef bool(rspamd_hdr_traverse_func_t)(const gchar *, const struct rspamd_mime_header *, void *);
+/**
+ * Traverse all headers in a table
+ * @param func
+ * @param ud
+ * @return
+ */
+bool rspamd_mime_headers_foreach(const struct rspamd_mime_headers_table *,
+								 rspamd_hdr_traverse_func_t func, void *ud);
+
+/**
+ * Strip rfc822 CFWS sequences from a string in place
+ * @param input input
+ * @param len length of the input
+ * @return new length of the input
+ */
+gsize rspamd_strip_smtp_comments_inplace(gchar *input, gsize len);
+
+/**
+ * Unfold header in place
+ * @param hdr header value
+ * @param len length of the header
+ * @return new unfolded length
+ */
+gsize rspamd_message_header_unfold_inplace(char *hdr, gsize len);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_MIME_HEADERS_H_ */
diff --git a/src/libmime/mime_parser.c b/src/libmime/mime_parser.c
new file mode 100644
index 0000000..217f0b8
--- /dev/null
+++ b/src/libmime/mime_parser.c
@@ -0,0 +1,1758 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#include "config.h"
+#include "task.h"
+#include "mime_parser.h"
+#include "mime_headers.h"
+#include "message.h"
+#include "multipattern.h"
+#include "contrib/libottery/ottery.h"
+#include "contrib/uthash/utlist.h"
+#include <openssl/cms.h>
+#include <openssl/pkcs7.h>
+#include "contrib/fastutf8/fastutf8.h"
+
+struct rspamd_mime_parser_lib_ctx {
+	struct rspamd_multipattern *mp_boundary;
+	guchar hkey[rspamd_cryptobox_SIPKEYBYTES]; /* Key for hashing */
+	guint key_usages;
+};
+
+struct rspamd_mime_parser_lib_ctx *lib_ctx = NULL;
+
+static const guint max_nested = 64;
+static const guint max_key_usages = 10000;
+
+#define msg_debug_mime(...) rspamd_conditional_debug_fast(NULL, task->from_addr,                                \
+														  rspamd_mime_log_id, "mime", task->task_pool->tag.uid, \
+														  RSPAMD_LOG_FUNC,                                      \
+														  __VA_ARGS__)
+
+INIT_LOG_MODULE(mime)
+
+#define RSPAMD_MIME_BOUNDARY_FLAG_CLOSED (1 << 0)
+#define RSPAMD_BOUNDARY_IS_CLOSED(b) ((b)->flags & RSPAMD_MIME_BOUNDARY_FLAG_CLOSED)
+
+struct rspamd_mime_boundary {
+	goffset boundary;
+	goffset start;
+	guint64 hash;
+	guint64 closed_hash;
+	gint flags;
+};
+
+struct rspamd_mime_parser_ctx {
+	GPtrArray *stack;   /* Stack of parts */
+	GArray *boundaries; /* Boundaries found in the whole message */
+	const gchar *start;
+	const gchar *pos;
+	const gchar *end;
+	struct rspamd_task *task;
+	guint nesting;
+};
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_multipart_part(struct rspamd_task *task,
+								 struct rspamd_mime_part *part,
+								 struct rspamd_mime_parser_ctx *st,
+								 GError **err);
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_message(struct rspamd_task *task,
+						  struct rspamd_mime_part *part,
+						  struct rspamd_mime_parser_ctx *st,
+						  GError **err);
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_normal_part(struct rspamd_task *task,
+							  struct rspamd_mime_part *part,
+							  struct rspamd_mime_parser_ctx *st,
+							  struct rspamd_content_type *ct,
+							  GError **err);
+
+static enum rspamd_mime_parse_error
+rspamd_mime_process_multipart_node(struct rspamd_task *task,
+								   struct rspamd_mime_parser_ctx *st,
+								   struct rspamd_mime_part *multipart,
+								   const gchar *start, const gchar *end,
+								   gboolean is_finished,
+								   GError **err);
+
+
+#define RSPAMD_MIME_QUARK (rspamd_mime_parser_quark())
+static GQuark
+rspamd_mime_parser_quark(void)
+{
+	return g_quark_from_static_string("mime-parser");
+}
+
+const gchar *
+rspamd_cte_to_string(enum rspamd_cte ct)
+{
+	const gchar *ret = "unknown";
+
+	switch (ct) {
+	case RSPAMD_CTE_7BIT:
+		ret = "7bit";
+		break;
+	case RSPAMD_CTE_8BIT:
+		ret = "8bit";
+		break;
+	case RSPAMD_CTE_QP:
+		ret = "quoted-printable";
+		break;
+	case RSPAMD_CTE_B64:
+		ret = "base64";
+		break;
+	case RSPAMD_CTE_UUE:
+		ret = "X-uuencode";
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+enum rspamd_cte
+rspamd_cte_from_string(const gchar *str)
+{
+	enum rspamd_cte ret = RSPAMD_CTE_UNKNOWN;
+
+	g_assert(str != NULL);
+
+	if (strcmp(str, "7bit") == 0) {
+		ret = RSPAMD_CTE_7BIT;
+	}
+	else if (strcmp(str, "8bit") == 0) {
+		ret = RSPAMD_CTE_8BIT;
+	}
+	else if (strcmp(str, "quoted-printable") == 0) {
+		ret = RSPAMD_CTE_QP;
+	}
+	else if (strcmp(str, "base64") == 0) {
+		ret = RSPAMD_CTE_B64;
+	}
+	else if (strcmp(str, "X-uuencode") == 0) {
+		ret = RSPAMD_CTE_UUE;
+	}
+	else if (strcmp(str, "uuencode") == 0) {
+		ret = RSPAMD_CTE_UUE;
+	}
+	else if (strcmp(str, "X-uue") == 0) {
+		ret = RSPAMD_CTE_UUE;
+	}
+
+	return ret;
+}
+
+static void
+rspamd_mime_parser_init_lib(void)
+{
+	lib_ctx = g_malloc0(sizeof(*lib_ctx));
+	lib_ctx->mp_boundary = rspamd_multipattern_create(RSPAMD_MULTIPATTERN_DEFAULT);
+	g_assert(lib_ctx->mp_boundary != NULL);
+	rspamd_multipattern_add_pattern(lib_ctx->mp_boundary, "\r--", 0);
+	rspamd_multipattern_add_pattern(lib_ctx->mp_boundary, "\n--", 0);
+
+	GError *err = NULL;
+	if (!rspamd_multipattern_compile(lib_ctx->mp_boundary, &err)) {
+		msg_err("fatal error: cannot compile multipattern for mime parser boundaries: %e", err);
+		g_error_free(err);
+		g_abort();
+	}
+	ottery_rand_bytes(lib_ctx->hkey, sizeof(lib_ctx->hkey));
+}
+
+static enum rspamd_cte
+rspamd_mime_parse_cte(const gchar *in, gsize len)
+{
+	guint64 h;
+	enum rspamd_cte ret = RSPAMD_CTE_UNKNOWN;
+
+	in = rspamd_string_len_strip(in, &len, " \t;,.+-#!`~'");
+	h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_XXHASH64,
+											in, len, 0xdeadbabe);
+
+	switch (h) {
+	case 0xCEDAA7056B4753F7ULL: /* 7bit */
+		ret = RSPAMD_CTE_7BIT;
+		break;
+	case 0x42E0745448B39FC1ULL: /* 8bit */
+	case 0x6B169E6B155BADC0ULL: /* binary */
+		ret = RSPAMD_CTE_8BIT;
+		break;
+	case 0x6D69A5BB02A633B0ULL: /* quoted-printable */
+		ret = RSPAMD_CTE_QP;
+		break;
+	case 0x96305588A76DC9A9ULL: /* base64 */
+	case 0x171029DE1B0423A9ULL: /* base-64 */
+		ret = RSPAMD_CTE_B64;
+		break;
+	case 0x420b54dc00d13cecULL: /* uuencode */
+	case 0x8df6700b8f6c4cf9ULL: /* x-uuencode */
+	case 0x41f725ec544356d3ULL: /* x-uue */
+		ret = RSPAMD_CTE_UUE;
+		break;
+	}
+
+	return ret;
+}
+
+static enum rspamd_cte
+rspamd_mime_part_get_cte_heuristic(struct rspamd_task *task,
+								   struct rspamd_mime_part *part)
+{
+	const guint check_len = 128;
+	guint real_len, nspaces = 0, neqsign = 0, n8bit = 0, nqpencoded = 0,
+					padeqsign = 0, nupper = 0, nlower = 0;
+	gboolean b64_chars = TRUE;
+	const guchar *p, *end;
+	enum rspamd_cte ret = RSPAMD_CTE_UNKNOWN;
+
+	real_len = MIN(check_len, part->raw_data.len);
+	p = (const guchar *) part->raw_data.begin;
+	end = p + part->raw_data.len;
+
+	while (p < end && g_ascii_isspace(*p)) {
+		p++;
+	}
+
+	if (end - p > sizeof("begin-base64 ")) {
+		const guchar *uue_start;
+
+		if (memcmp(p, "begin ", sizeof("begin ") - 1) == 0) {
+			uue_start = p + sizeof("begin ") - 1;
+
+			while (uue_start < end && g_ascii_isspace(*uue_start)) {
+				uue_start++;
+			}
+
+			if (uue_start < end && g_ascii_isdigit(*uue_start)) {
+				return RSPAMD_CTE_UUE;
+			}
+		}
+		else if (memcmp(p, "begin-base64 ", sizeof("begin-base64 ") - 1) == 0) {
+			uue_start = p + sizeof("begin ") - 1;
+
+			while (uue_start < end && g_ascii_isspace(*uue_start)) {
+				uue_start++;
+			}
+
+			if (uue_start < end && g_ascii_isdigit(*uue_start)) {
+				return RSPAMD_CTE_UUE;
+			}
+		}
+	}
+
+	/* Skip trailing spaces */
+	while (end > p && g_ascii_isspace(*(end - 1))) {
+		end--;
+	}
+
+	if (end > p + 2) {
+		if (*(end - 1) == '=') {
+			padeqsign++;
+			end--;
+		}
+
+		if (*(end - 1) == '=') {
+			padeqsign++;
+			end--;
+		}
+	}
+
+	/* Adjust end to analyse only first characters */
+	if (end - p > real_len) {
+		end = p + real_len;
+	}
+
+	while (p < end) {
+		if (*p == ' ') {
+			nspaces++;
+		}
+		else if (*p == '=') {
+			b64_chars = FALSE; /* Eqsign must not be inside base64 */
+			neqsign++;
+			p++;
+
+			if (p + 2 < end && g_ascii_isxdigit(*p) && g_ascii_isxdigit(*(p + 1))) {
+				p++;
+				nqpencoded++;
+			}
+
+			continue;
+		}
+		else if (*p >= 0x80) {
+			n8bit++;
+			b64_chars = FALSE;
+		}
+		else if (!(g_ascii_isalnum(*p) || *p == '/' || *p == '+')) {
+			b64_chars = FALSE;
+		}
+		else if (g_ascii_isupper(*p)) {
+			nupper++;
+		}
+		else if (g_ascii_islower(*p)) {
+			nlower++;
+		}
+
+		p++;
+	}
+
+	if (b64_chars && neqsign <= 2 && nspaces == 0) {
+		/* Need more thinking */
+
+		if (part->raw_data.len > 80) {
+			if (padeqsign > 0) {
+				ret = RSPAMD_CTE_B64;
+			}
+			else {
+				/* We have a large piece of data with no spaces and base64
+				 * symbols only, no padding is detected as well...
+				 *
+				 * There is a small chance that our first 128 characters
+				 * are either some garbage or it is a base64 with no padding
+				 * (e.g. when it is not needed)
+				 */
+				if (nupper > 1 && nlower > 1) {
+					/*
+					 * We have both uppercase and lowercase letters, so it can be
+					 * base64
+					 */
+					ret = RSPAMD_CTE_B64;
+				}
+				else {
+					ret = RSPAMD_CTE_7BIT;
+				}
+			}
+		}
+		else {
+
+			if (((end - (const guchar *) part->raw_data.begin) + padeqsign) % 4 == 0) {
+				if (padeqsign == 0) {
+					/*
+					 * It can be either base64 or plain text, hard to say
+					 * Let's assume that if we have > 1 uppercase it is
+					 * likely base64
+					 */
+					if (nupper > 1 && nlower > 1) {
+						ret = RSPAMD_CTE_B64;
+					}
+					else {
+						ret = RSPAMD_CTE_7BIT;
+					}
+				}
+				else {
+					ret = RSPAMD_CTE_B64;
+				}
+			}
+			else {
+				/* No way */
+				if (padeqsign == 1 || padeqsign == 2) {
+					ret = RSPAMD_CTE_B64;
+				}
+				else {
+					ret = RSPAMD_CTE_7BIT;
+				}
+			}
+		}
+	}
+	else if (n8bit == 0) {
+		if (neqsign > 2 && nqpencoded > 2) {
+			ret = RSPAMD_CTE_QP;
+		}
+		else {
+			ret = RSPAMD_CTE_7BIT;
+		}
+	}
+	else {
+		ret = RSPAMD_CTE_8BIT;
+	}
+
+	msg_debug_mime("detected cte: %s", rspamd_cte_to_string(ret));
+
+	return ret;
+}
+
+static void
+rspamd_mime_part_get_cte(struct rspamd_task *task,
+						 struct rspamd_mime_headers_table *hdrs,
+						 struct rspamd_mime_part *part,
+						 gboolean apply_heuristic)
+{
+	struct rspamd_mime_header *hdr, *cur;
+	enum rspamd_cte cte = RSPAMD_CTE_UNKNOWN;
+	gboolean parent_propagated = FALSE;
+
+	hdr = rspamd_message_get_header_from_hash(hdrs, "Content-Transfer-Encoding", FALSE);
+
+	if (hdr == NULL) {
+		if (part->parent_part && part->parent_part->cte != RSPAMD_CTE_UNKNOWN &&
+			!(part->parent_part->flags & RSPAMD_MIME_PART_MISSING_CTE)) {
+			part->cte = part->parent_part->cte;
+			parent_propagated = TRUE;
+
+			goto check_cte;
+		}
+
+		if (apply_heuristic) {
+			part->cte = rspamd_mime_part_get_cte_heuristic(task, part);
+			msg_info_task("detected missing CTE for part as: %s",
+						  rspamd_cte_to_string(part->cte));
+		}
+
+		part->flags |= RSPAMD_MIME_PART_MISSING_CTE;
+	}
+	else {
+		DL_FOREACH(hdr, cur)
+		{
+			gsize hlen;
+			gchar lc_buf[128];
+
+			hlen = rspamd_snprintf(lc_buf, sizeof(lc_buf), "%s", cur->value);
+			rspamd_str_lc(lc_buf, hlen);
+			cte = rspamd_mime_parse_cte(lc_buf, hlen);
+
+			if (cte != RSPAMD_CTE_UNKNOWN) {
+				part->cte = cte;
+				break;
+			}
+		}
+
+	check_cte:
+		if (apply_heuristic) {
+			if (part->cte == RSPAMD_CTE_UNKNOWN) {
+				part->cte = rspamd_mime_part_get_cte_heuristic(task, part);
+
+				msg_info_task("corrected bad CTE for part to: %s",
+							  rspamd_cte_to_string(part->cte));
+			}
+			else if (part->cte == RSPAMD_CTE_B64 ||
+					 part->cte == RSPAMD_CTE_QP) {
+				/* Additionally check sanity */
+				cte = rspamd_mime_part_get_cte_heuristic(task, part);
+
+				if (cte == RSPAMD_CTE_8BIT) {
+					msg_info_task(
+						"incorrect cte specified for part: %s, %s detected",
+						rspamd_cte_to_string(part->cte),
+						rspamd_cte_to_string(cte));
+					part->cte = cte;
+					part->flags |= RSPAMD_MIME_PART_BAD_CTE;
+				}
+				else if (cte != part->cte && parent_propagated) {
+					part->cte = cte;
+					msg_info_task("detected missing CTE for part as: %s",
+								  rspamd_cte_to_string(part->cte));
+				}
+			}
+			else {
+				msg_debug_mime("processed cte: %s",
+							   rspamd_cte_to_string(cte));
+			}
+		}
+		else {
+			msg_debug_mime("processed cte: %s", rspamd_cte_to_string(cte));
+		}
+	}
+}
+static void
+rspamd_mime_part_get_cd(struct rspamd_task *task, struct rspamd_mime_part *part)
+{
+	struct rspamd_mime_header *hdr, *cur;
+	struct rspamd_content_disposition *cd = NULL;
+	rspamd_ftok_t srch;
+	struct rspamd_content_type_param *found;
+
+	hdr = rspamd_message_get_header_from_hash(part->raw_headers,
+											  "Content-Disposition", FALSE);
+
+
+	if (hdr == NULL) {
+		cd = rspamd_mempool_alloc0(task->task_pool, sizeof(*cd));
+		cd->type = RSPAMD_CT_INLINE;
+
+		/* We can also have content disposition definitions in Content-Type */
+		if (part->ct && part->ct->attrs) {
+			RSPAMD_FTOK_ASSIGN(&srch, "name");
+			found = g_hash_table_lookup(part->ct->attrs, &srch);
+
+			if (!found) {
+				RSPAMD_FTOK_ASSIGN(&srch, "filename");
+				found = g_hash_table_lookup(part->ct->attrs, &srch);
+			}
+
+			if (found) {
+				cd->type = RSPAMD_CT_ATTACHMENT;
+				memcpy(&cd->filename, &found->value, sizeof(cd->filename));
+			}
+		}
+	}
+	else {
+		DL_FOREACH(hdr, cur)
+		{
+			gsize hlen;
+			cd = NULL;
+
+			if (cur->value) {
+				hlen = strlen(cur->value);
+				cd = rspamd_content_disposition_parse(cur->value, hlen,
+													  task->task_pool);
+			}
+
+			if (cd) {
+				/* We still need to check filename */
+				if (cd->filename.len == 0) {
+					if (part->ct && part->ct->attrs) {
+						RSPAMD_FTOK_ASSIGN(&srch, "name");
+						found = g_hash_table_lookup(part->ct->attrs, &srch);
+
+						if (!found) {
+							RSPAMD_FTOK_ASSIGN(&srch, "filename");
+							found = g_hash_table_lookup(part->ct->attrs, &srch);
+						}
+
+						if (found) {
+							cd->type = RSPAMD_CT_ATTACHMENT;
+							memcpy(&cd->filename, &found->value,
+								   sizeof(cd->filename));
+						}
+					}
+				}
+
+				msg_debug_mime("processed content disposition: %s, file: \"%T\"",
+							   cd->lc_data, &cd->filename);
+				break;
+			}
+			else if (part->ct) {
+				/*
+				 * Even in case of malformed Content-Disposition, we can still
+				 * fall back to Content-Type
+				 */
+				cd = rspamd_mempool_alloc0(task->task_pool, sizeof(*cd));
+				cd->type = RSPAMD_CT_INLINE;
+
+				/* We can also have content disposition definitions in Content-Type */
+				if (part->ct->attrs) {
+					RSPAMD_FTOK_ASSIGN(&srch, "name");
+					found = g_hash_table_lookup(part->ct->attrs, &srch);
+
+					if (!found) {
+						RSPAMD_FTOK_ASSIGN(&srch, "filename");
+						found = g_hash_table_lookup(part->ct->attrs, &srch);
+					}
+
+					if (found) {
+						cd->type = RSPAMD_CT_ATTACHMENT;
+						memcpy(&cd->filename, &found->value, sizeof(cd->filename));
+					}
+				}
+			}
+		}
+	}
+
+	part->cd = cd;
+}
+
+void rspamd_mime_parser_calc_digest(struct rspamd_mime_part *part)
+{
+	/* Blake2b applied to string 'rspamd' */
+	static const guchar hash_key[] = {
+		0xef,
+		0x43,
+		0xae,
+		0x80,
+		0xcc,
+		0x8d,
+		0xc3,
+		0x4c,
+		0x6f,
+		0x1b,
+		0xd6,
+		0x18,
+		0x1b,
+		0xae,
+		0x87,
+		0x74,
+		0x0c,
+		0xca,
+		0xf7,
+		0x8e,
+		0x5f,
+		0x2e,
+		0x54,
+		0x32,
+		0xf6,
+		0x79,
+		0xb9,
+		0x27,
+		0x26,
+		0x96,
+		0x20,
+		0x92,
+		0x70,
+		0x07,
+		0x85,
+		0xeb,
+		0x83,
+		0xf7,
+		0x89,
+		0xe0,
+		0xd7,
+		0x32,
+		0x2a,
+		0xd2,
+		0x1a,
+		0x64,
+		0x41,
+		0xef,
+		0x49,
+		0xff,
+		0xc3,
+		0x8c,
+		0x54,
+		0xf9,
+		0x67,
+		0x74,
+		0x30,
+		0x1e,
+		0x70,
+		0x2e,
+		0xb7,
+		0x12,
+		0x09,
+		0xfe,
+	};
+
+	if (part->parsed_data.len > 0) {
+		rspamd_cryptobox_hash(part->digest,
+							  part->parsed_data.begin, part->parsed_data.len,
+							  hash_key, sizeof(hash_key));
+	}
+}
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_normal_part(struct rspamd_task *task,
+							  struct rspamd_mime_part *part,
+							  struct rspamd_mime_parser_ctx *st,
+							  struct rspamd_content_type *ct,
+							  GError **err)
+{
+	rspamd_fstring_t *parsed;
+	gssize r;
+
+	g_assert(part != NULL);
+
+	rspamd_mime_part_get_cte(task, part->raw_headers, part,
+							 part->ct && !(part->ct->flags & RSPAMD_CONTENT_TYPE_MESSAGE));
+	rspamd_mime_part_get_cd(task, part);
+
+	switch (part->cte) {
+	case RSPAMD_CTE_7BIT:
+	case RSPAMD_CTE_8BIT:
+	case RSPAMD_CTE_UNKNOWN:
+		if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_MISSING)) {
+			if (part->cte != RSPAMD_CTE_7BIT) {
+				/* We have something that has a missing content-type,
+				 * but it has non-7bit characters.
+				 *
+				 * In theory, it is very unsafe to process it as a text part
+				 * as we unlikely get some sane result
+				 */
+
+				/*
+				 * On the other hand, there is an evidence that some
+				 * emails actually rely on that.
+				 * So we apply an expensive hack here:
+				 * if there are no 8bit characters -OR- the content is valid
+				 * UTF8, we can still imply Content-Type == text/plain
+				 */
+
+				if (rspamd_str_has_8bit(part->raw_data.begin, part->raw_data.len) &&
+					!rspamd_fast_utf8_validate(part->raw_data.begin, part->raw_data.len)) {
+					part->ct->flags &= ~RSPAMD_CONTENT_TYPE_TEXT;
+					part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+				}
+			}
+		}
+
+		if (part->ct && (part->ct->flags & RSPAMD_CONTENT_TYPE_TEXT)) {
+			/* Need to copy text as we have couple of in-place change functions */
+			parsed = rspamd_fstring_sized_new(part->raw_data.len);
+			parsed->len = part->raw_data.len;
+			memcpy(parsed->str, part->raw_data.begin, parsed->len);
+			part->parsed_data.begin = parsed->str;
+			part->parsed_data.len = parsed->len;
+			rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+			rspamd_mempool_add_destructor(task->task_pool,
+										  (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+		}
+		else {
+			part->parsed_data.begin = part->raw_data.begin;
+			part->parsed_data.len = part->raw_data.len;
+		}
+		break;
+	case RSPAMD_CTE_QP:
+		parsed = rspamd_fstring_sized_new(part->raw_data.len);
+		r = rspamd_decode_qp_buf(part->raw_data.begin, part->raw_data.len,
+								 parsed->str, parsed->allocated);
+		if (r != -1) {
+			parsed->len = r;
+			part->parsed_data.begin = parsed->str;
+			part->parsed_data.len = parsed->len;
+			rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+			rspamd_mempool_add_destructor(task->task_pool,
+										  (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+		}
+		else {
+			msg_err_task("invalid quoted-printable encoded part, assume 8bit");
+			if (part->ct) {
+				part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+			}
+			part->cte = RSPAMD_CTE_8BIT;
+			memcpy(parsed->str, part->raw_data.begin, part->raw_data.len);
+			parsed->len = part->raw_data.len;
+			part->parsed_data.begin = parsed->str;
+			part->parsed_data.len = parsed->len;
+			rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+			rspamd_mempool_add_destructor(task->task_pool,
+										  (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+		}
+		break;
+	case RSPAMD_CTE_B64:
+		parsed = rspamd_fstring_sized_new(part->raw_data.len / 4 * 3 + 12);
+		rspamd_cryptobox_base64_decode(part->raw_data.begin,
+									   part->raw_data.len,
+									   parsed->str, &parsed->len);
+		part->parsed_data.begin = parsed->str;
+		part->parsed_data.len = parsed->len;
+		rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+		rspamd_mempool_add_destructor(task->task_pool,
+									  (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+		break;
+	case RSPAMD_CTE_UUE:
+		parsed = rspamd_fstring_sized_new(part->raw_data.len / 4 * 3 + 12);
+		r = rspamd_decode_uue_buf(part->raw_data.begin, part->raw_data.len,
+								  parsed->str, parsed->allocated);
+		rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+		rspamd_mempool_add_destructor(task->task_pool,
+									  (rspamd_mempool_destruct_t) rspamd_fstring_free, parsed);
+		if (r != -1) {
+			parsed->len = r;
+			part->parsed_data.begin = parsed->str;
+			part->parsed_data.len = parsed->len;
+		}
+		else {
+			msg_err_task("invalid uuencoding in encoded part, assume 8bit");
+			if (part->ct) {
+				part->ct->flags |= RSPAMD_CONTENT_TYPE_BROKEN;
+			}
+			part->cte = RSPAMD_CTE_8BIT;
+			parsed->len = MIN(part->raw_data.len, parsed->allocated);
+			memcpy(parsed->str, part->raw_data.begin, parsed->len);
+			rspamd_mempool_notify_alloc(task->task_pool, parsed->len);
+			part->parsed_data.begin = parsed->str;
+			part->parsed_data.len = parsed->len;
+		}
+		break;
+	default:
+		g_assert_not_reached();
+	}
+
+	part->part_number = MESSAGE_FIELD(task, parts)->len;
+	part->urls = g_ptr_array_new();
+	g_ptr_array_add(MESSAGE_FIELD(task, parts), part);
+	msg_debug_mime("parsed data part %T/%T of length %z (%z orig), %s cte",
+				   &part->ct->type, &part->ct->subtype, part->parsed_data.len,
+				   part->raw_data.len, rspamd_cte_to_string(part->cte));
+	rspamd_mime_parser_calc_digest(part);
+
+	if (ct && (ct->flags & RSPAMD_CONTENT_TYPE_SMIME)) {
+		CMS_ContentInfo *cms;
+		const unsigned char *der_beg = part->parsed_data.begin;
+		cms = d2i_CMS_ContentInfo(NULL, &der_beg, part->parsed_data.len);
+
+		if (cms) {
+			const ASN1_OBJECT *asn_ct = CMS_get0_eContentType(cms);
+			int ct_nid = OBJ_obj2nid(asn_ct);
+
+			if (ct_nid == NID_pkcs7_data) {
+				BIO *bio = BIO_new_mem_buf(part->parsed_data.begin,
+										   part->parsed_data.len);
+
+				PKCS7 *p7;
+				p7 = d2i_PKCS7_bio(bio, NULL);
+
+				if (p7) {
+					ct_nid = OBJ_obj2nid(p7->type);
+
+					if (ct_nid == NID_pkcs7_signed) {
+						PKCS7 *p7_signed_content = p7->d.sign->contents;
+
+						ct_nid = OBJ_obj2nid(p7_signed_content->type);
+
+						if (ct_nid == NID_pkcs7_data && p7_signed_content->d.data) {
+							int ret;
+
+							msg_debug_mime("found an additional part inside of "
+										   "smime structure of type %T/%T; length=%d",
+										   &ct->type, &ct->subtype, p7_signed_content->d.data->length);
+							/*
+							 * Since ASN.1 structures are freed, we need to copy
+							 * the content
+							 */
+							gchar *cpy = rspamd_mempool_alloc(task->task_pool,
+															  p7_signed_content->d.data->length);
+							memcpy(cpy, p7_signed_content->d.data->data,
+								   p7_signed_content->d.data->length);
+							ret = rspamd_mime_process_multipart_node(task,
+																	 st, NULL,
+																	 cpy, cpy + p7_signed_content->d.data->length,
+																	 TRUE, err);
+
+							PKCS7_free(p7);
+							BIO_free(bio);
+							CMS_ContentInfo_free(cms);
+
+							return ret;
+						}
+					}
+
+					PKCS7_free(p7);
+				}
+
+				BIO_free(bio);
+			}
+
+			CMS_ContentInfo_free(cms);
+		}
+	}
+
+	return RSPAMD_MIME_PARSE_OK;
+}
+
+struct rspamd_mime_multipart_cbdata {
+	struct rspamd_task *task;
+	struct rspamd_mime_part *multipart;
+	struct rspamd_mime_parser_ctx *st;
+	const gchar *part_start;
+	rspamd_ftok_t *cur_boundary;
+	guint64 bhash;
+	GError **err;
+};
+
+static enum rspamd_mime_parse_error
+rspamd_mime_process_multipart_node(struct rspamd_task *task,
+								   struct rspamd_mime_parser_ctx *st,
+								   struct rspamd_mime_part *multipart,
+								   const gchar *start, const gchar *end,
+								   gboolean is_finished,
+								   GError **err)
+{
+	struct rspamd_content_type *ct, *sel = NULL;
+	struct rspamd_mime_header *hdr = NULL, *cur;
+	struct rspamd_mime_part *npart;
+	GString str;
+	goffset hdr_pos, body_pos;
+	enum rspamd_mime_parse_error ret = RSPAMD_MIME_PARSE_FATAL;
+
+
+	str.str = (gchar *) start;
+	str.len = end - start;
+
+	if (*start == '\n' || *start == '\r') {
+		/*
+		 * We have a part that starts from newline which means that
+		 * there are completely no headers in this part,
+		 * hence we assume it as a text part
+		 */
+		hdr_pos = 0;
+		body_pos = 0;
+
+		if (!is_finished) {
+			/* Ignore garbage */
+			const gchar *p = start;
+			gboolean seen_something = FALSE;
+
+			while (p < end) {
+				if (g_ascii_isalnum(*p)) {
+					seen_something = TRUE;
+					break;
+				}
+				p++;
+			}
+
+			if (!seen_something) {
+				return RSPAMD_MIME_PARSE_NO_PART;
+			}
+		}
+	}
+	else {
+		hdr_pos = rspamd_string_find_eoh(&str, &body_pos);
+	}
+
+	npart = rspamd_mempool_alloc0(task->task_pool,
+								  sizeof(struct rspamd_mime_part));
+	npart->parent_part = multipart;
+	npart->raw_headers = rspamd_message_headers_new();
+	npart->headers_order = NULL;
+
+	if (multipart) {
+		if (multipart->specific.mp->children == NULL) {
+			multipart->specific.mp->children = g_ptr_array_sized_new(2);
+		}
+
+		g_ptr_array_add(multipart->specific.mp->children, npart);
+	}
+
+	if (hdr_pos > 0 && hdr_pos < str.len) {
+		npart->raw_headers_str = str.str;
+		npart->raw_headers_len = hdr_pos;
+		npart->raw_data.begin = start + body_pos;
+		npart->raw_data.len = (end - start) - body_pos;
+
+		if (npart->raw_headers_len > 0) {
+			rspamd_mime_headers_process(task, npart->raw_headers,
+										&npart->headers_order,
+										npart->raw_headers_str,
+										npart->raw_headers_len,
+										FALSE);
+
+			/* Preserve the natural order */
+			if (npart->headers_order) {
+				LL_REVERSE2(npart->headers_order, ord_next);
+			}
+		}
+
+		hdr = rspamd_message_get_header_from_hash(npart->raw_headers,
+												  "Content-Type", FALSE);
+	}
+	else {
+		npart->raw_headers_str = 0;
+		npart->raw_headers_len = 0;
+		npart->raw_data.begin = start;
+		npart->raw_data.len = end - start;
+	}
+
+
+	if (hdr != NULL) {
+
+		DL_FOREACH(hdr, cur)
+		{
+			ct = rspamd_content_type_parse(cur->value, strlen(cur->value),
+										   task->task_pool);
+
+			/* Here we prefer multipart content-type or any content-type */
+			if (ct) {
+				if (sel == NULL) {
+					sel = ct;
+				}
+				else if (ct->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
+					sel = ct;
+				}
+			}
+		}
+	}
+
+	if (sel == NULL) {
+		sel = rspamd_mempool_alloc0(task->task_pool, sizeof(*sel));
+		RSPAMD_FTOK_ASSIGN(&sel->type, "text");
+		RSPAMD_FTOK_ASSIGN(&sel->subtype, "plain");
+	}
+
+	npart->ct = sel;
+
+	if (sel->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
+		st->nesting++;
+		g_ptr_array_add(st->stack, npart);
+		npart->part_type = RSPAMD_MIME_PART_MULTIPART;
+		npart->specific.mp = rspamd_mempool_alloc0(task->task_pool,
+												   sizeof(struct rspamd_mime_multipart));
+		memcpy(&npart->specific.mp->boundary, &sel->orig_boundary,
+			   sizeof(rspamd_ftok_t));
+		ret = rspamd_mime_parse_multipart_part(task, npart, st, err);
+	}
+	else if (sel->flags & RSPAMD_CONTENT_TYPE_MESSAGE) {
+		st->nesting++;
+		g_ptr_array_add(st->stack, npart);
+		npart->part_type = RSPAMD_MIME_PART_MESSAGE;
+
+		if ((ret = rspamd_mime_parse_normal_part(task, npart, st, sel, err)) == RSPAMD_MIME_PARSE_OK) {
+			ret = rspamd_mime_parse_message(task, npart, st, err);
+		}
+	}
+	else {
+		ret = rspamd_mime_parse_normal_part(task, npart, st, sel, err);
+	}
+
+	return ret;
+}
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_multipart_cb(struct rspamd_task *task,
+							   struct rspamd_mime_part *multipart,
+							   struct rspamd_mime_parser_ctx *st,
+							   struct rspamd_mime_multipart_cbdata *cb,
+							   struct rspamd_mime_boundary *b)
+{
+	const gchar *pos = st->start + b->boundary;
+	enum rspamd_mime_parse_error ret;
+
+	task = cb->task;
+
+	/* Now check boundary */
+	if (!cb->part_start) {
+		cb->part_start = st->start + b->start;
+		st->pos = cb->part_start;
+	}
+	else {
+		/*
+		 * We have seen the start of the boundary,
+		 * but it might be unsuitable (e.g. in broken headers)
+		 */
+		if (cb->part_start < pos && cb->cur_boundary) {
+
+			if ((ret = rspamd_mime_process_multipart_node(task, cb->st,
+														  cb->multipart, cb->part_start, pos, TRUE, cb->err)) != RSPAMD_MIME_PARSE_OK) {
+				return ret;
+			}
+
+			if (b->start > 0) {
+				/* Go towards the next part */
+				cb->part_start = st->start + b->start;
+				cb->st->pos = cb->part_start;
+			}
+		}
+		else {
+			/* We have an empty boundary, do nothing */
+		}
+	}
+
+	return RSPAMD_MIME_PARSE_OK;
+}
+
+static enum rspamd_mime_parse_error
+rspamd_multipart_boundaries_filter(struct rspamd_task *task,
+								   struct rspamd_mime_part *multipart,
+								   struct rspamd_mime_parser_ctx *st,
+								   struct rspamd_mime_multipart_cbdata *cb)
+{
+	struct rspamd_mime_boundary *cur;
+	goffset last_offset;
+	guint i, sel = 0;
+	enum rspamd_mime_parse_error ret;
+
+	last_offset = (multipart->raw_data.begin - st->start) +
+				  multipart->raw_data.len;
+
+	/* Find the first offset suitable for this part */
+	for (i = 0; i < st->boundaries->len; i++) {
+		cur = &g_array_index(st->boundaries, struct rspamd_mime_boundary, i);
+
+		if (cur->start >= multipart->raw_data.begin - st->start) {
+			if (cb->cur_boundary) {
+				/* Check boundary */
+				msg_debug_mime("compare %L and %L (and %L)",
+							   cb->bhash, cur->hash, cur->closed_hash);
+
+				if (cb->bhash == cur->hash) {
+					sel = i;
+					break;
+				}
+				else if (cb->bhash == cur->closed_hash) {
+					/* Not a closing element in fact */
+					cur->flags &= ~(RSPAMD_MIME_BOUNDARY_FLAG_CLOSED);
+					cur->hash = cur->closed_hash;
+					sel = i;
+					break;
+				}
+			}
+			else {
+				/* Set current boundary */
+				cb->cur_boundary = rspamd_mempool_alloc(task->task_pool,
+														sizeof(rspamd_ftok_t));
+				cb->cur_boundary->begin = st->start + cur->boundary;
+				cb->cur_boundary->len = 0;
+				cb->bhash = cur->hash;
+				sel = i;
+				break;
+			}
+		}
+	}
+
+	/* Now we can go forward with boundaries that are same to what we have */
+	for (i = sel; i < st->boundaries->len; i++) {
+		cur = &g_array_index(st->boundaries, struct rspamd_mime_boundary, i);
+
+		if (cur->boundary > last_offset) {
+			break;
+		}
+
+		if (cur->hash == cb->bhash || cur->closed_hash == cb->bhash) {
+			if ((ret = rspamd_mime_parse_multipart_cb(task, multipart, st,
+													  cb, cur)) != RSPAMD_MIME_PARSE_OK) {
+				return ret;
+			}
+
+			if (cur->closed_hash == cb->bhash) {
+				/* We have again fake closed hash */
+				cur->flags &= ~(RSPAMD_MIME_BOUNDARY_FLAG_CLOSED);
+				cur->hash = cur->closed_hash;
+			}
+
+			if (RSPAMD_BOUNDARY_IS_CLOSED(cur)) {
+				/* We also might check the next boundary... */
+				if (i < st->boundaries->len - 1) {
+					cur = &g_array_index(st->boundaries,
+										 struct rspamd_mime_boundary, i + 1);
+
+					if (cur->hash == cb->bhash) {
+						continue;
+					}
+					else if (cur->closed_hash == cb->bhash) {
+						/* We have again fake closed hash */
+						cur->flags &= ~(RSPAMD_MIME_BOUNDARY_FLAG_CLOSED);
+						cur->hash = cur->closed_hash;
+						continue;
+					}
+				}
+
+				break;
+			}
+		}
+	}
+
+	if (i == st->boundaries->len && cb->cur_boundary) {
+		/* Process the last part */
+		struct rspamd_mime_boundary fb;
+
+		fb.boundary = last_offset;
+		fb.start = -1;
+
+		if ((ret = rspamd_mime_parse_multipart_cb(task, multipart, st,
+												  cb, &fb)) != RSPAMD_MIME_PARSE_OK) {
+			return ret;
+		}
+	}
+
+	return RSPAMD_MIME_PARSE_OK;
+}
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_multipart_part(struct rspamd_task *task,
+								 struct rspamd_mime_part *part,
+								 struct rspamd_mime_parser_ctx *st,
+								 GError **err)
+{
+	struct rspamd_mime_multipart_cbdata cbdata;
+	enum rspamd_mime_parse_error ret;
+
+	if (st->nesting > max_nested) {
+		g_set_error(err, RSPAMD_MIME_QUARK, E2BIG, "Nesting level is too high: %d",
+					st->nesting);
+		return RSPAMD_MIME_PARSE_NESTING;
+	}
+
+	part->part_number = MESSAGE_FIELD(task, parts)->len;
+	part->urls = g_ptr_array_new();
+	g_ptr_array_add(MESSAGE_FIELD(task, parts), part);
+	st->nesting++;
+	rspamd_mime_part_get_cte(task, part->raw_headers, part, FALSE);
+
+	st->pos = part->raw_data.begin;
+	cbdata.multipart = part;
+	cbdata.task = task;
+	cbdata.st = st;
+	cbdata.part_start = NULL;
+	cbdata.err = err;
+
+	if (part->ct->boundary.len > 0) {
+		/* We know our boundary */
+		cbdata.cur_boundary = &part->ct->boundary;
+		rspamd_cryptobox_siphash((guchar *) &cbdata.bhash,
+								 cbdata.cur_boundary->begin, cbdata.cur_boundary->len,
+								 lib_ctx->hkey);
+		msg_debug_mime("hash: %T -> %L", cbdata.cur_boundary, cbdata.bhash);
+	}
+	else {
+		/* Guess boundary */
+		cbdata.cur_boundary = NULL;
+		cbdata.bhash = 0;
+	}
+
+	ret = rspamd_multipart_boundaries_filter(task, part, st, &cbdata);
+	/* Cleanup stack */
+	st->nesting--;
+	g_ptr_array_remove_index_fast(st->stack, st->stack->len - 1);
+
+	return ret;
+}
+
+/* Process boundary like structures in a message */
+static gint
+rspamd_mime_preprocess_cb(struct rspamd_multipattern *mp,
+						  guint strnum,
+						  gint match_start,
+						  gint match_pos,
+						  const gchar *text,
+						  gsize len,
+						  void *context)
+{
+	const gchar *end = text + len, *p = text + match_pos, *bend;
+	gsize blen;
+	gboolean closing = FALSE;
+	struct rspamd_mime_boundary b;
+	struct rspamd_mime_parser_ctx *st = context;
+	struct rspamd_task *task;
+
+	task = st->task;
+
+	if (G_LIKELY(p < end)) {
+
+		blen = 0;
+
+		while (p < end) {
+			if (*p == '\r' || *p == '\n') {
+				break;
+			}
+
+			blen++;
+			p++;
+		}
+
+		if (blen > 0) {
+			/* We have found something like boundary */
+			p = text + match_pos;
+			bend = p + blen - 1;
+
+			if (*bend == '-') {
+				/* We need to verify last -- */
+				if (bend > p + 1 && *(bend - 1) == '-') {
+					closing = TRUE;
+					bend--;
+					blen -= 2;
+				}
+				else {
+					/* Not a closing boundary somehow, e.g. if a boundary=='-' */
+					bend++;
+				}
+			}
+			else {
+				bend++;
+			}
+
+			while (bend < end) {
+				if (*bend == '\r') {
+					bend++;
+
+					/* \r\n */
+					if (bend < end && *bend == '\n') {
+						bend++;
+					}
+				}
+				else if (*bend == '\n') {
+					/* \n */
+					bend++;
+				}
+				else if (g_ascii_isspace(*bend)) {
+					/* Spaces in the same line, skip them */
+					bend++;
+					continue;
+				}
+
+				break;
+			}
+
+			b.boundary = p - st->start - 2;
+			b.start = bend - st->start;
+
+			/* Small optimisation as boundaries are usually short strings */
+			gchar *lc_copy, lc_copy_buf[128];
+
+			if (blen + 2 < sizeof(lc_copy_buf)) {
+				lc_copy = lc_copy_buf;
+			}
+			else {
+				lc_copy = g_malloc(blen + 2);
+			}
+
+			if (closing) {
+				memcpy(lc_copy, p, blen + 2);
+				rspamd_str_lc(lc_copy, blen + 2);
+			}
+			else {
+				memcpy(lc_copy, p, blen);
+				rspamd_str_lc(lc_copy, blen);
+			}
+
+			rspamd_cryptobox_siphash((guchar *) &b.hash, lc_copy, blen,
+									 lib_ctx->hkey);
+			msg_debug_mime("normal hash: %*s -> %L, %d boffset, %d data offset",
+						   (gint) blen, lc_copy, b.hash, (int) b.boundary, (int) b.start);
+
+			if (closing) {
+				b.flags = RSPAMD_MIME_BOUNDARY_FLAG_CLOSED;
+				rspamd_cryptobox_siphash((guchar *) &b.closed_hash, lc_copy,
+										 blen + 2,
+										 lib_ctx->hkey);
+				msg_debug_mime("closing hash: %*s -> %L, %d boffset, %d data offset",
+							   (gint) blen + 2, lc_copy,
+							   b.closed_hash,
+							   (int) b.boundary, (int) b.start);
+			}
+			else {
+				b.flags = 0;
+				b.closed_hash = 0;
+			}
+
+			/* Check if a string has been allocated on the heap */
+			if (blen + 2 >= sizeof(lc_copy_buf)) {
+				g_free(lc_copy);
+			}
+			g_array_append_val(st->boundaries, b);
+		}
+	}
+
+	return 0;
+}
+
+static goffset
+rspamd_mime_parser_headers_heuristic(GString *input, goffset *body_start)
+{
+	const gsize default_max_len = 76;
+	gsize max_len = MIN(input->len, default_max_len);
+	const gchar *p, *end;
+	enum {
+		st_before_colon = 0,
+		st_colon,
+		st_spaces_after_colon,
+		st_value,
+		st_error
+	} state = st_before_colon;
+
+	p = input->str;
+	end = p + max_len;
+
+	while (p < end) {
+		switch (state) {
+		case st_before_colon:
+			if (G_UNLIKELY(*p == ':')) {
+				state = st_colon;
+			}
+			else if (G_UNLIKELY(!g_ascii_isgraph(*p))) {
+				state = st_error;
+			}
+
+			p++;
+			break;
+		case st_colon:
+			if (g_ascii_isspace(*p)) {
+				state = st_spaces_after_colon;
+			}
+			else {
+				state = st_value;
+			}
+			p++;
+			break;
+		case st_spaces_after_colon:
+			if (!g_ascii_isspace(*p)) {
+				state = st_value;
+			}
+			p++;
+			break;
+		case st_value:
+			/* We accept any value */
+			goto end;
+			break;
+		case st_error:
+			return (-1);
+			break;
+		}
+	}
+
+end:
+	if (state == st_value) {
+		if (body_start) {
+			*body_start = input->len;
+		}
+
+		return input->len;
+	}
+
+	return (-1);
+}
+
+static void
+rspamd_mime_preprocess_message(struct rspamd_task *task,
+							   struct rspamd_mime_part *top,
+							   struct rspamd_mime_parser_ctx *st)
+{
+
+	if (top->raw_data.begin >= st->pos) {
+		rspamd_multipattern_lookup(lib_ctx->mp_boundary,
+								   top->raw_data.begin - 1,
+								   top->raw_data.len + 1,
+								   rspamd_mime_preprocess_cb, st, NULL);
+	}
+	else {
+		rspamd_multipattern_lookup(lib_ctx->mp_boundary,
+								   st->pos,
+								   st->end - st->pos,
+								   rspamd_mime_preprocess_cb, st, NULL);
+	}
+}
+
+static void
+rspamd_mime_parse_stack_free(struct rspamd_mime_parser_ctx *st)
+{
+	if (st) {
+		g_ptr_array_free(st->stack, TRUE);
+		g_array_free(st->boundaries, TRUE);
+		g_free(st);
+	}
+}
+
+static enum rspamd_mime_parse_error
+rspamd_mime_parse_message(struct rspamd_task *task,
+						  struct rspamd_mime_part *part,
+						  struct rspamd_mime_parser_ctx *st,
+						  GError **err)
+{
+	struct rspamd_content_type *ct, *sel = NULL;
+	struct rspamd_mime_header *hdr = NULL, *cur;
+	const gchar *pbegin, *p;
+	gsize plen, len;
+	struct rspamd_mime_part *npart;
+	goffset hdr_pos, body_pos;
+	guint i;
+	enum rspamd_mime_parse_error ret = RSPAMD_MIME_PARSE_OK;
+	GString str;
+	struct rspamd_mime_parser_ctx *nst = st;
+
+	if (st->nesting > max_nested) {
+		g_set_error(err, RSPAMD_MIME_QUARK, E2BIG, "Nesting level is too high: %d",
+					st->nesting);
+		return RSPAMD_MIME_PARSE_NESTING;
+	}
+
+	/* Allocate real part */
+	npart = rspamd_mempool_alloc0(task->task_pool,
+								  sizeof(struct rspamd_mime_part));
+
+	if (part == NULL) {
+		/* Top level message */
+		p = task->msg.begin;
+		len = task->msg.len;
+
+		str.str = (gchar *) p;
+		str.len = len;
+
+		hdr_pos = rspamd_string_find_eoh(&str, &body_pos);
+
+		if (hdr_pos > 0 && hdr_pos < str.len) {
+
+			MESSAGE_FIELD(task, raw_headers_content).begin = str.str;
+			MESSAGE_FIELD(task, raw_headers_content).len = hdr_pos;
+			MESSAGE_FIELD(task, raw_headers_content).body_start = str.str + body_pos;
+
+			if (MESSAGE_FIELD(task, raw_headers_content).len > 0) {
+				rspamd_mime_headers_process(task,
+											MESSAGE_FIELD(task, raw_headers),
+											&MESSAGE_FIELD(task, headers_order),
+											MESSAGE_FIELD(task, raw_headers_content).begin,
+											MESSAGE_FIELD(task, raw_headers_content).len,
+											TRUE);
+				npart->raw_headers = rspamd_message_headers_ref(
+					MESSAGE_FIELD(task, raw_headers));
+
+				/* Preserve the natural order */
+				if (MESSAGE_FIELD(task, headers_order)) {
+					LL_REVERSE2(MESSAGE_FIELD(task, headers_order), ord_next);
+				}
+			}
+
+			hdr = rspamd_message_get_header_from_hash(
+				MESSAGE_FIELD(task, raw_headers),
+				"Content-Type", FALSE);
+		}
+		else {
+			/* First apply heuristic, maybe we have just headers */
+			hdr_pos = rspamd_mime_parser_headers_heuristic(&str, &body_pos);
+
+			if (hdr_pos > 0 && hdr_pos <= str.len) {
+				MESSAGE_FIELD(task, raw_headers_content).begin = str.str;
+				MESSAGE_FIELD(task, raw_headers_content).len = hdr_pos;
+				MESSAGE_FIELD(task, raw_headers_content).body_start = str.str +
+																	  body_pos;
+
+				if (MESSAGE_FIELD(task, raw_headers_content).len > 0) {
+					rspamd_mime_headers_process(task,
+												MESSAGE_FIELD(task, raw_headers),
+												&MESSAGE_FIELD(task, headers_order),
+												MESSAGE_FIELD(task, raw_headers_content).begin,
+												MESSAGE_FIELD(task, raw_headers_content).len,
+												TRUE);
+					npart->raw_headers = rspamd_message_headers_ref(
+						MESSAGE_FIELD(task, raw_headers));
+
+					/* Preserve the natural order */
+					if (MESSAGE_FIELD(task, headers_order)) {
+						LL_REVERSE2(MESSAGE_FIELD(task, headers_order), ord_next);
+					}
+				}
+
+				hdr = rspamd_message_get_header_from_hash(
+					MESSAGE_FIELD(task, raw_headers),
+					"Content-Type", FALSE);
+				task->flags |= RSPAMD_TASK_FLAG_BROKEN_HEADERS;
+			}
+			else {
+				body_pos = 0;
+			}
+		}
+
+		pbegin = st->start + body_pos;
+		plen = st->end - pbegin;
+		npart->headers_order = NULL;
+	}
+	else {
+		/*
+		 * Here are dragons:
+		 * We allocate new parser context as we need to shift pointers
+		 */
+		nst = g_malloc0(sizeof(*st));
+		nst->stack = g_ptr_array_sized_new(4);
+		nst->boundaries = g_array_sized_new(FALSE, FALSE,
+											sizeof(struct rspamd_mime_boundary), 8);
+		nst->start = part->parsed_data.begin;
+		nst->end = nst->start + part->parsed_data.len;
+		nst->pos = nst->start;
+		nst->task = st->task;
+		nst->nesting = st->nesting;
+		st->nesting++;
+
+		str.str = (gchar *) part->parsed_data.begin;
+		str.len = part->parsed_data.len;
+
+		hdr_pos = rspamd_string_find_eoh(&str, &body_pos);
+		npart->raw_headers = rspamd_message_headers_new();
+		npart->headers_order = NULL;
+
+		if (hdr_pos > 0 && hdr_pos < str.len) {
+			npart->raw_headers_str = str.str;
+			npart->raw_headers_len = hdr_pos;
+			npart->raw_data.begin = str.str + body_pos;
+
+			if (npart->raw_headers_len > 0) {
+				rspamd_mime_headers_process(task,
+											npart->raw_headers,
+											&npart->headers_order,
+											npart->raw_headers_str,
+											npart->raw_headers_len,
+											FALSE);
+
+				/* Preserve the natural order */
+				if (npart->headers_order) {
+					LL_REVERSE2(npart->headers_order, ord_next);
+				}
+			}
+
+			hdr = rspamd_message_get_header_from_hash(npart->raw_headers,
+													  "Content-Type", FALSE);
+		}
+		else {
+			body_pos = 0;
+		}
+
+		pbegin = part->parsed_data.begin + body_pos;
+		plen = part->parsed_data.len - body_pos;
+	}
+
+	npart->raw_data.begin = pbegin;
+	npart->raw_data.len = plen;
+	npart->parent_part = part;
+
+	if (hdr == NULL) {
+		sel = NULL;
+	}
+	else {
+		DL_FOREACH(hdr, cur)
+		{
+			ct = rspamd_content_type_parse(cur->value, strlen(cur->value),
+										   task->task_pool);
+
+			/* Here we prefer multipart content-type or any content-type */
+			if (ct) {
+				if (sel == NULL) {
+					sel = ct;
+				}
+				else if (ct->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
+					sel = ct;
+				}
+			}
+		}
+	}
+
+	if (sel == NULL) {
+		/* For messages we automatically assume plaintext */
+		msg_info_task("cannot find content-type for a message, assume text/plain");
+		sel = rspamd_mempool_alloc0(task->task_pool, sizeof(*sel));
+		sel->flags = RSPAMD_CONTENT_TYPE_TEXT | RSPAMD_CONTENT_TYPE_MISSING;
+		RSPAMD_FTOK_ASSIGN(&sel->type, "text");
+		RSPAMD_FTOK_ASSIGN(&sel->subtype, "plain");
+	}
+
+	npart->ct = sel;
+
+	if ((part == NULL || nst != st) &&
+		(sel->flags & (RSPAMD_CONTENT_TYPE_MULTIPART | RSPAMD_CONTENT_TYPE_MESSAGE))) {
+		/* Not a trivial message, need to preprocess */
+		rspamd_mime_preprocess_message(task, npart, nst);
+	}
+
+	if (sel->flags & RSPAMD_CONTENT_TYPE_MULTIPART) {
+		g_ptr_array_add(nst->stack, npart);
+		nst->nesting++;
+		npart->part_type = RSPAMD_MIME_PART_MULTIPART;
+		npart->specific.mp = rspamd_mempool_alloc0(task->task_pool,
+												   sizeof(struct rspamd_mime_multipart));
+		memcpy(&npart->specific.mp->boundary, &sel->orig_boundary,
+			   sizeof(rspamd_ftok_t));
+		ret = rspamd_mime_parse_multipart_part(task, npart, nst, err);
+	}
+	else if (sel->flags & RSPAMD_CONTENT_TYPE_MESSAGE) {
+		if ((ret = rspamd_mime_parse_normal_part(task, npart, nst, sel, err)) == RSPAMD_MIME_PARSE_OK) {
+			npart->part_type = RSPAMD_MIME_PART_MESSAGE;
+			ret = rspamd_mime_parse_message(task, npart, nst, err);
+		}
+	}
+	else {
+		ret = rspamd_mime_parse_normal_part(task, npart, nst, sel, err);
+	}
+
+	if (ret != RSPAMD_MIME_PARSE_OK) {
+		return ret;
+	}
+
+	if (part && st->stack->len > 0) {
+		/* Remove message part from the parent stack */
+		g_ptr_array_remove_index_fast(st->stack, st->stack->len - 1);
+		st->nesting--;
+	}
+
+	/* Process leftovers for boundaries */
+	if (nst->boundaries) {
+		struct rspamd_mime_boundary *boundary, *start_boundary = NULL,
+											   *end_boundary = NULL;
+		goffset cur_offset = nst->pos - nst->start,
+				end_offset = st->end - st->start;
+		guint sel_idx = 0;
+
+		for (;;) {
+			start_boundary = NULL;
+
+			for (i = sel_idx; i < nst->boundaries->len; i++) {
+				boundary = &g_array_index(nst->boundaries,
+										  struct rspamd_mime_boundary, i);
+
+				if (boundary->start > cur_offset &&
+					boundary->boundary < end_offset &&
+					!RSPAMD_BOUNDARY_IS_CLOSED(boundary)) {
+					start_boundary = boundary;
+					sel_idx = i;
+					break;
+				}
+			}
+
+			if (start_boundary) {
+				const gchar *start, *end;
+
+				if (nst->boundaries->len > sel_idx + 1) {
+					end_boundary = &g_array_index(nst->boundaries,
+												  struct rspamd_mime_boundary, sel_idx + 1);
+					end = nst->start + end_boundary->boundary;
+				}
+				else {
+					end = nst->end;
+				}
+
+				sel_idx++;
+
+				start = nst->start + start_boundary->start;
+
+				if (end > start &&
+					(ret = rspamd_mime_process_multipart_node(task, nst,
+															  NULL, start, end, FALSE, err)) != RSPAMD_MIME_PARSE_OK) {
+
+					if (nst != st) {
+						rspamd_mime_parse_stack_free(nst);
+					}
+
+					if (ret == RSPAMD_MIME_PARSE_NO_PART) {
+						return RSPAMD_MIME_PARSE_OK;
+					}
+
+					return ret;
+				}
+			}
+			else {
+				break;
+			}
+		}
+	}
+
+	if (nst != st) {
+		rspamd_mime_parse_stack_free(nst);
+	}
+
+	return ret;
+}
+
+enum rspamd_mime_parse_error
+rspamd_mime_parse_task(struct rspamd_task *task, GError **err)
+{
+	struct rspamd_mime_parser_ctx *st;
+	enum rspamd_mime_parse_error ret = RSPAMD_MIME_PARSE_OK;
+
+	if (lib_ctx == NULL) {
+		rspamd_mime_parser_init_lib();
+	}
+
+	if (++lib_ctx->key_usages > max_key_usages) {
+		/* Regenerate siphash key */
+		ottery_rand_bytes(lib_ctx->hkey, sizeof(lib_ctx->hkey));
+		lib_ctx->key_usages = 0;
+	}
+
+	st = g_malloc0(sizeof(*st));
+	st->stack = g_ptr_array_sized_new(4);
+	st->pos = MESSAGE_FIELD(task, raw_headers_content).body_start;
+	st->end = task->msg.begin + task->msg.len;
+	st->boundaries = g_array_sized_new(FALSE, FALSE,
+									   sizeof(struct rspamd_mime_boundary), 8);
+	st->task = task;
+
+	if (st->pos == NULL) {
+		st->pos = task->msg.begin;
+	}
+
+	st->start = task->msg.begin;
+	ret = rspamd_mime_parse_message(task, NULL, st, err);
+	rspamd_mime_parse_stack_free(st);
+
+	return ret;
+}
diff --git a/src/libmime/mime_parser.h b/src/libmime/mime_parser.h
new file mode 100644
index 0000000..aa77b2b
--- /dev/null
+++ b/src/libmime/mime_parser.h
@@ -0,0 +1,46 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_MIME_PARSER_H_
+#define SRC_LIBMIME_MIME_PARSER_H_
+
+#include "config.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_mime_part;
+
+enum rspamd_mime_parse_error {
+	RSPAMD_MIME_PARSE_OK = 0,
+	RSPAMD_MIME_PARSE_FATAL,
+	RSPAMD_MIME_PARSE_NESTING,
+	RSPAMD_MIME_PARSE_NO_PART,
+};
+
+enum rspamd_mime_parse_error rspamd_mime_parse_task(struct rspamd_task *task,
+													GError **err);
+
+void rspamd_mime_parser_calc_digest(struct rspamd_mime_part *part);
+
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_MIME_PARSER_H_ */
diff --git a/src/libmime/mime_string.cxx b/src/libmime/mime_string.cxx
new file mode 100644
index 0000000..e818e64
--- /dev/null
+++ b/src/libmime/mime_string.cxx
@@ -0,0 +1,167 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+#include "mime_string.hxx"
+#include "unicode/uchar.h"
+
+TEST_SUITE("mime_string")
+{
+	using namespace rspamd::mime;
+	TEST_CASE("mime_string unfiltered ctors")
+	{
+		SUBCASE("empty")
+		{
+			mime_string st;
+			CHECK(st.size() == 0);
+			CHECK(st == "");
+		}
+		SUBCASE("unfiltered valid")
+		{
+			mime_string st{std::string_view("abcd")};
+			CHECK(st == "abcd");
+		}
+		SUBCASE("unfiltered zero character")
+		{
+			mime_string st{"abc\0d", 5};
+			CHECK(st.has_zeroes());
+			CHECK(st == "abcd");
+		}
+		SUBCASE("unfiltered invalid character - middle")
+		{
+			mime_string st{std::string("abc\234d")};
+			CHECK(st.has_invalid());
+			CHECK(st == "abc\uFFFDd");
+		}
+		SUBCASE("unfiltered invalid character - end")
+		{
+			mime_string st{std::string("abc\234")};
+			CHECK(st.has_invalid());
+			CHECK(st == "abc\uFFFD");
+		}
+		SUBCASE("unfiltered invalid character - start")
+		{
+			mime_string st{std::string("\234abc")};
+			CHECK(st.has_invalid());
+			CHECK(st == "\uFFFDabc");
+		}
+	}
+
+	TEST_CASE("mime_string filtered ctors")
+	{
+		auto print_filter = [](UChar32 inp) -> UChar32 {
+			if (!u_isprint(inp)) {
+				return 0;
+			}
+
+			return inp;
+		};
+
+		auto tolower_filter = [](UChar32 inp) -> UChar32 {
+			return u_tolower(inp);
+		};
+
+		SUBCASE("empty")
+		{
+			mime_string st{std::string_view(""), tolower_filter};
+			CHECK(st.size() == 0);
+			CHECK(st == "");
+		}
+		SUBCASE("filtered valid")
+		{
+			mime_string st{std::string("AbCdУ"), tolower_filter};
+			CHECK(st == "abcdу");
+		}
+		SUBCASE("filtered invalid + filtered")
+		{
+			mime_string st{std::string("abcd\234\1"), print_filter};
+			CHECK(st == "abcd\uFFFD");
+		}
+	}
+	TEST_CASE("mime_string assign")
+	{
+		SUBCASE("assign from valid")
+		{
+			mime_string st;
+
+			CHECK(st.assign_if_valid(std::string("test")));
+			CHECK(st == "test");
+		}
+		SUBCASE("assign from invalid")
+		{
+			mime_string st;
+
+			CHECK(!st.assign_if_valid(std::string("test\234t")));
+			CHECK(st == "");
+		}
+	}
+
+	TEST_CASE("mime_string iterators")
+	{
+
+		SUBCASE("unfiltered iterator ascii")
+		{
+			auto in = std::string("abcd");
+			mime_string st{in};
+			CHECK(st == "abcd");
+
+			int i = 0;
+			for (auto &&c: st) {
+				CHECK(c == in[i++]);
+			}
+		}
+
+		SUBCASE("unfiltered iterator utf8")
+		{
+			auto in = std::string("тест");
+			UChar32 ucs[4] = {1090, 1077, 1089, 1090};
+			mime_string st{in};
+			CHECK(st == "тест");
+
+			int i = 0;
+			for (auto &&c: st) {
+				CHECK(c == ucs[i++]);
+			}
+			CHECK(i == sizeof(ucs) / sizeof(ucs[0]));
+		}
+
+		SUBCASE("unfiltered raw iterator ascii")
+		{
+			auto in = std::string("abcd");
+			mime_string st{in};
+			CHECK(st == "abcd");
+
+			int i = 0;
+			for (auto it = st.raw_begin(); it != st.raw_end(); ++it) {
+				CHECK(*it == in[i++]);
+			}
+		}
+
+		SUBCASE("unfiltered raw iterator utf8")
+		{
+			auto in = std::string("тест");
+			mime_string st{in};
+			CHECK(st == "тест");
+
+			int i = 0;
+			for (auto it = st.raw_begin(); it != st.raw_end(); ++it) {
+				CHECK(*it == in[i++]);
+			}
+			CHECK(i == in.size());
+		}
+	}
+}
\ No newline at end of file
diff --git a/src/libmime/mime_string.hxx b/src/libmime/mime_string.hxx
new file mode 100644
index 0000000..7476816
--- /dev/null
+++ b/src/libmime/mime_string.hxx
@@ -0,0 +1,670 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef RSPAMD_MIME_STRING_HXX
+#define RSPAMD_MIME_STRING_HXX
+#pragma once
+
+#include <algorithm>
+#include <string>
+#include <string_view>
+#include <memory>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
+#include <iosfwd>
+#include "libutil/mem_pool.h"
+#include "function2/function2.hpp"
+#include "unicode/utf8.h"
+#include "contrib/fastutf8/fastutf8.h"
+
+namespace rspamd::mime {
+/*
+ * The motivation for another string is to have utf8 valid string replacing
+ * all bad things with FFFFD replacement character and filtering \0 and other
+ * strange stuff defined by policies.
+ * This string always exclude \0 characters and ignore them! This is how MUA acts,
+ * and we also store a flag about bad characters.
+ * Mime string iterators are always const, so the underlying storage should not
+ * be modified externally.
+ */
+template<class T = char, class Allocator = std::allocator<T>,
+		 class Functor = fu2::function_view<UChar32(UChar32)>>
+class basic_mime_string;
+
+using mime_string = basic_mime_string<char>;
+using mime_pool_string = basic_mime_string<char, mempool_allocator<char>>;
+
+/* Helpers for type safe flags */
+enum class mime_string_flags : std::uint8_t {
+	MIME_STRING_DEFAULT = 0,
+	MIME_STRING_SEEN_ZEROES = 0x1 << 0,
+	MIME_STRING_SEEN_INVALID = 0x1 << 1,
+};
+
+constexpr mime_string_flags operator|(mime_string_flags lhs, mime_string_flags rhs)
+{
+	using ut = std::underlying_type<mime_string_flags>::type;
+	return static_cast<mime_string_flags>(static_cast<ut>(lhs) | static_cast<ut>(rhs));
+}
+
+constexpr mime_string_flags operator&(mime_string_flags lhs, mime_string_flags rhs)
+{
+	using ut = std::underlying_type<mime_string_flags>::type;
+	return static_cast<mime_string_flags>(static_cast<ut>(lhs) & static_cast<ut>(rhs));
+}
+
+constexpr bool operator!(mime_string_flags fl)
+{
+	return fl == mime_string_flags::MIME_STRING_DEFAULT;
+}
+
+// Codepoint iterator base class
+template<typename Container, bool Raw = false>
+struct iterator_base {
+	template<typename, typename, typename>
+	friend class basic_mime_string;
+
+public:
+	using value_type = typename Container::value_type;
+	using difference_type = typename Container::difference_type;
+	using codepoint_type = typename Container::codepoint_type;
+	using reference_type = codepoint_type;
+	using iterator_category = std::bidirectional_iterator_tag;
+
+	bool operator==(const iterator_base &it) const noexcept
+	{
+		return idx == it.idx;
+	}
+
+	bool operator!=(const iterator_base &it) const noexcept
+	{
+		return idx != it.idx;
+	}
+
+	iterator_base(difference_type index, Container *instance) noexcept
+		: idx(index), cont_instance(instance)
+	{
+	}
+	iterator_base() noexcept = default;
+	iterator_base(const iterator_base &) noexcept = default;
+
+	iterator_base &operator=(const iterator_base &) noexcept = default;
+
+	Container *get_instance() const noexcept
+	{
+		return cont_instance;
+	}
+
+	codepoint_type get_value() const noexcept
+	{
+		auto i = idx;
+		codepoint_type uc;
+		U8_NEXT_UNSAFE(cont_instance->data(), i, uc);
+		return uc;
+	}
+
+protected:
+	difference_type idx;
+	Container *cont_instance = nullptr;
+
+protected:
+	void advance(difference_type n) noexcept
+	{
+		if (n > 0) {
+			U8_FWD_N_UNSAFE(cont_instance->data(), idx, n);
+		}
+		else if (n < 0) {
+			U8_BACK_N_UNSAFE(cont_instance->data(), idx, (-n));
+		}
+	}
+	void increment() noexcept
+	{
+		codepoint_type uc;
+		U8_NEXT_UNSAFE(cont_instance->data(), idx, uc);
+	}
+
+	void decrement() noexcept
+	{
+		codepoint_type uc;
+		U8_PREV_UNSAFE(cont_instance->data(), idx, uc);
+	}
+};
+
+// Partial spec for raw Byte-based iterator base
+template<typename Container>
+struct iterator_base<Container, true> {
+	template<typename, typename, typename>
+	friend class basic_string;
+
+public:
+	using value_type = typename Container::value_type;
+	using difference_type = typename Container::difference_type;
+	using reference_type = value_type;
+	using iterator_category = std::bidirectional_iterator_tag;
+
+	bool operator==(const iterator_base &it) const noexcept
+	{
+		return idx == it.idx;
+	}
+	bool operator!=(const iterator_base &it) const noexcept
+	{
+		return idx != it.idx;
+	}
+
+	iterator_base(difference_type index, Container *instance) noexcept
+		: idx(index), cont_instance(instance)
+	{
+	}
+
+	iterator_base() noexcept = default;
+	iterator_base(const iterator_base &) noexcept = default;
+	iterator_base &operator=(const iterator_base &) noexcept = default;
+	Container *get_instance() const noexcept
+	{
+		return cont_instance;
+	}
+
+	value_type get_value() const noexcept
+	{
+		return cont_instance->get_storage().at(idx);
+	}
+
+protected:
+	difference_type idx;
+	Container *cont_instance = nullptr;
+
+protected:
+	//! Advance the iterator n times (negative values allowed!)
+	void advance(difference_type n) noexcept
+	{
+		idx += n;
+	}
+
+	void increment() noexcept
+	{
+		idx++;
+	}
+	void decrement() noexcept
+	{
+		idx--;
+	}
+};
+
+template<typename Container, bool Raw>
+struct iterator;
+template<typename Container, bool Raw>
+struct const_iterator;
+
+template<typename Container, bool Raw = false>
+struct iterator : iterator_base<Container, Raw> {
+	iterator(typename iterator_base<Container, Raw>::difference_type index, Container *instance) noexcept
+		: iterator_base<Container, Raw>(index, instance)
+	{
+	}
+	iterator() noexcept = default;
+	iterator(const iterator &) noexcept = default;
+
+	iterator &operator=(const iterator &) noexcept = default;
+	/* Disallow creating from const_iterator */
+	iterator(const const_iterator<Container, Raw> &) = delete;
+
+	/* Prefix */
+	iterator &operator++() noexcept
+	{
+		this->increment();
+		return *this;
+	}
+
+	/* Postfix */
+	iterator operator++(int) noexcept
+	{
+		iterator tmp{this->idx, this->cont_instance};
+		this->increment();
+		return tmp;
+	}
+
+	/* Prefix */
+	iterator &operator--() noexcept
+	{
+		this->decrement();
+		return *this;
+	}
+
+	/* Postfix */
+	iterator operator--(int) noexcept
+	{
+		iterator tmp{this->idx, this->cont_instance};
+		this->decrement();
+		return tmp;
+	}
+
+	iterator operator+(typename iterator_base<Container, Raw>::difference_type n) const noexcept
+	{
+		iterator it{*this};
+		it.advance(n);
+		return it;
+	}
+
+	iterator &operator+=(typename iterator_base<Container, Raw>::difference_type n) noexcept
+	{
+		this->advance(n);
+		return *this;
+	}
+
+	iterator operator-(typename iterator_base<Container, Raw>::difference_type n) const noexcept
+	{
+		iterator it{*this};
+		it.advance(-n);
+		return it;
+	}
+
+	iterator &operator-=(typename iterator_base<Container, Raw>::difference_type n) noexcept
+	{
+		this->advance(-n);
+		return *this;
+	}
+
+	typename iterator::reference_type operator*() const noexcept
+	{
+		return this->get_value();
+	}
+};
+
+template<class CharT, class Allocator, class Functor>
+class basic_mime_string : private Allocator {
+public:
+	using storage_type = std::basic_string<CharT, std::char_traits<CharT>, Allocator>;
+	using view_type = std::basic_string_view<CharT, std::char_traits<CharT>>;
+	using filter_type = Functor;
+	using codepoint_type = UChar32;
+	using value_type = CharT;
+	using difference_type = std::ptrdiff_t;
+	using iterator = rspamd::mime::iterator<basic_mime_string, false>;
+	using raw_iterator = rspamd::mime::iterator<basic_mime_string, true>;
+	/* Ctors */
+	basic_mime_string() noexcept
+		: Allocator()
+	{
+	}
+	explicit basic_mime_string(const Allocator &alloc) noexcept
+		: Allocator(alloc)
+	{
+	}
+	explicit basic_mime_string(filter_type &&filt, const Allocator &alloc = Allocator()) noexcept
+		: Allocator(alloc), filter_func(std::move(filt))
+	{
+	}
+
+	basic_mime_string(const CharT *str, std::size_t sz, const Allocator &alloc = Allocator()) noexcept
+		: Allocator(alloc)
+	{
+		append_c_string_unfiltered(str, sz);
+	}
+
+	basic_mime_string(const storage_type &st,
+					  const Allocator &alloc = Allocator()) noexcept
+		: basic_mime_string(st.data(), st.size(), alloc)
+	{
+	}
+
+	basic_mime_string(const view_type &st,
+					  const Allocator &alloc = Allocator()) noexcept
+		: basic_mime_string(st.data(), st.size(), alloc)
+	{
+	}
+	/* Explicit move ctor */
+	basic_mime_string(basic_mime_string &&other) noexcept
+	{
+		*this = std::move(other);
+	}
+
+
+	/**
+	 * Creates a string with a filter function. It is calee responsibility to
+	 * ensure that the filter functor survives long enough to work with a string
+	 * @param str
+	 * @param sz
+	 * @param filt
+	 * @param alloc
+	 */
+	basic_mime_string(const CharT *str, std::size_t sz,
+					  filter_type &&filt,
+					  const Allocator &alloc = Allocator()) noexcept
+		: Allocator(alloc),
+		  filter_func(std::move(filt))
+	{
+		append_c_string_filtered(str, sz);
+	}
+
+	basic_mime_string(const storage_type &st,
+					  filter_type &&filt,
+					  const Allocator &alloc = Allocator()) noexcept
+		: basic_mime_string(st.data(), st.size(), std::move(filt), alloc)
+	{
+	}
+	basic_mime_string(const view_type &st,
+					  filter_type &&filt,
+					  const Allocator &alloc = Allocator()) noexcept
+		: basic_mime_string(st.data(), st.size(), std::move(filt), alloc)
+	{
+	}
+
+	/* It seems some libc++ implementations still perform copy, this might fix them */
+	basic_mime_string &operator=(basic_mime_string &&other)
+	{
+		storage = std::move(other.storage);
+		filter_func = std::move(other.filter_func);
+
+		return *this;
+	}
+
+	constexpr auto size() const noexcept -> std::size_t
+	{
+		return storage.size();
+	}
+
+	constexpr auto data() const noexcept -> const CharT *
+	{
+		return storage.data();
+	}
+
+	constexpr auto has_zeroes() const noexcept -> bool
+	{
+		return !!(flags & mime_string_flags::MIME_STRING_SEEN_ZEROES);
+	}
+
+	constexpr auto has_invalid() const noexcept -> bool
+	{
+		return !!(flags & mime_string_flags::MIME_STRING_SEEN_INVALID);
+	}
+
+	/**
+	 * Assign mime string from another string using move operation if a source string
+	 * is utf8 valid.
+	 * If this function returns false, then ownership has not been transferred
+	 * and the `other` string is unmodified as well as the storage
+	 * @param other
+	 * @return
+	 */
+	[[nodiscard]] auto assign_if_valid(storage_type &&other) -> bool
+	{
+		if (filter_func) {
+			/* No way */
+			return false;
+		}
+		if (rspamd_fast_utf8_validate((const unsigned char *) other.data(), other.size()) == 0) {
+			std::swap(storage, other);
+
+			return true;
+		}
+
+		return false;
+	}
+
+	/**
+	 * Copy to the internal storage discarding the contained value
+	 * @param other
+	 * @return
+	 */
+	auto assign_copy(const view_type &other)
+	{
+		storage.clear();
+
+		if (filter_func) {
+			append_c_string_filtered(other.data(), other.size());
+		}
+		else {
+			append_c_string_unfiltered(other.data(), other.size());
+		}
+	}
+	auto assign_copy(const storage_type &other)
+	{
+		storage.clear();
+
+		if (filter_func) {
+			append_c_string_filtered(other.data(), other.size());
+		}
+		else {
+			append_c_string_unfiltered(other.data(), other.size());
+		}
+	}
+	auto assign_copy(const basic_mime_string &other)
+	{
+		storage.clear();
+
+		if (filter_func) {
+			append_c_string_filtered(other.data(), other.size());
+		}
+		else {
+			append_c_string_unfiltered(other.data(), other.size());
+		}
+	}
+
+	/* Mutators */
+	auto append(const CharT *str, std::size_t size) -> std::size_t
+	{
+		if (filter_func) {
+			return append_c_string_filtered(str, size);
+		}
+		else {
+			return append_c_string_unfiltered(str, size);
+		}
+	}
+	auto append(const storage_type &other) -> std::size_t
+	{
+		return append(other.data(), other.size());
+	}
+	auto append(const view_type &other) -> std::size_t
+	{
+		return append(other.data(), other.size());
+	}
+
+	auto ltrim(const view_type &what) -> void
+	{
+		auto it = std::find_if(storage.begin(), storage.end(),
+							   [&what](CharT c) {
+								   return !std::any_of(what.begin(), what.end(), [&c](CharT sc) { return sc == c; });
+							   });
+		storage.erase(storage.begin(), it);
+	}
+
+	auto rtrim(const view_type &what) -> void
+	{
+		auto it = std::find_if(storage.rbegin(), storage.rend(),
+							   [&what](CharT c) {
+								   return !std::any_of(what.begin(), what.end(), [&c](CharT sc) { return sc == c; });
+							   });
+		storage.erase(it.base(), storage.end());
+	}
+
+	auto trim(const view_type &what) -> void
+	{
+		ltrim(what);
+		rtrim(what);
+	}
+
+	/* Comparison */
+	auto operator==(const basic_mime_string &other)
+	{
+		return other.storage == storage;
+	}
+	auto operator==(const storage_type &other)
+	{
+		return other == storage;
+	}
+	auto operator==(const view_type &other)
+	{
+		return other == storage;
+	}
+	auto operator==(const CharT *other)
+	{
+		if (other == NULL) {
+			return false;
+		}
+		auto olen = strlen(other);
+		if (storage.size() == olen) {
+			return memcmp(storage.data(), other, olen) == 0;
+		}
+
+		return false;
+	}
+
+	/* Iterators */
+	inline auto begin() noexcept -> iterator
+	{
+		return {0, this};
+	}
+
+	inline auto raw_begin() noexcept -> raw_iterator
+	{
+		return {0, this};
+	}
+
+	inline auto end() noexcept -> iterator
+	{
+		return {(difference_type) size(), this};
+	}
+
+	inline auto raw_end() noexcept -> raw_iterator
+	{
+		return {(difference_type) size(), this};
+	}
+
+	/* Utility */
+	inline auto get_storage() const noexcept -> const storage_type &
+	{
+		return storage;
+	}
+
+	inline auto as_view() const noexcept -> view_type
+	{
+		return view_type{storage};
+	}
+
+	constexpr CharT operator[](std::size_t pos) const noexcept
+	{
+		return storage[pos];
+	}
+	constexpr CharT at(std::size_t pos) const
+	{
+		return storage.at(pos);
+	}
+	constexpr bool empty() const noexcept
+	{
+		return storage.empty();
+	}
+
+
+	/* For doctest stringify */
+	friend std::ostream &operator<<(std::ostream &os, const CharT &value)
+	{
+		os << value.storage;
+		return os;
+	}
+
+private:
+	mime_string_flags flags = mime_string_flags::MIME_STRING_DEFAULT;
+	storage_type storage;
+	filter_type filter_func;
+
+	auto append_c_string_unfiltered(const CharT *str, std::size_t len) -> std::size_t
+	{
+		/* This is fast path */
+		const auto *p = str;
+		const auto *end = str + len;
+		std::int32_t err_offset;// We have to use int32_t here as old libicu is brain-damaged
+		auto orig_size = storage.size();
+
+		storage.reserve(len + storage.size());
+
+		if (memchr(str, 0, len) != NULL) {
+			/* Fallback to slow path */
+			flags = flags | mime_string_flags::MIME_STRING_SEEN_ZEROES;
+			return append_c_string_filtered(str, len);
+		}
+
+		while (p < end && len > 0 &&
+			   (err_offset = rspamd_fast_utf8_validate((const unsigned char *) p, len)) > 0) {
+			auto cur_offset = err_offset - 1;
+			storage.append(p, cur_offset);
+
+			while (cur_offset < len) {
+				auto tmp = cur_offset;
+				UChar32 uc;
+
+				U8_NEXT(p, cur_offset, len, uc);
+
+				if (uc < 0) {
+					storage.append("\uFFFD");
+					flags = flags | mime_string_flags::MIME_STRING_SEEN_INVALID;
+				}
+				else {
+					cur_offset = tmp;
+					break;
+				}
+			}
+
+			p += cur_offset;
+			len = end - p;
+		}
+
+		storage.append(p, len);
+		return storage.size() - orig_size;
+	}
+
+	auto append_c_string_filtered(const CharT *str, std::size_t len) -> std::size_t
+	{
+		std::int32_t i = 0;// We have to use int32_t here as old libicu is brain-damaged
+		UChar32 uc;
+		char tmp[4];
+		auto orig_size = storage.size();
+		/* Slow path */
+
+		storage.reserve(len + storage.size());
+
+		while (i < len) {
+			U8_NEXT(str, i, len, uc);
+
+			if (uc < 0) {
+				/* Replace with 0xFFFD */
+				storage.append("\uFFFD");
+				flags = flags | mime_string_flags::MIME_STRING_SEEN_INVALID;
+			}
+			else {
+				if (filter_func) {
+					uc = filter_func(uc);
+				}
+
+				if (uc == 0) {
+					/* Special case, ignore it */
+					flags = flags | mime_string_flags::MIME_STRING_SEEN_ZEROES;
+				}
+				else {
+					std::int32_t o = 0;
+					U8_APPEND_UNSAFE(tmp, o, uc);
+					storage.append(tmp, o);
+				}
+			}
+		}
+
+		return storage.size() - orig_size;
+	}
+};
+
+}// namespace rspamd::mime
+
+
+#endif//RSPAMD_MIME_STRING_HXX
diff --git a/src/libmime/received.cxx b/src/libmime/received.cxx
new file mode 100644
index 0000000..dc16d9b
--- /dev/null
+++ b/src/libmime/received.cxx
@@ -0,0 +1,1017 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "libserver/url.h"
+#include "lua/lua_common.h"
+#include "libserver/cfg_file.h"
+#include "libserver/mempool_vars_internal.h"
+#include "mime_string.hxx"
+#include "smtp_parsers.h"
+#include "message.h"
+#include "received.hxx"
+#include "frozen/string.h"
+#include "frozen/unordered_map.h"
+
+namespace rspamd::mime {
+
+enum class received_part_type {
+	RSPAMD_RECEIVED_PART_FROM,
+	RSPAMD_RECEIVED_PART_BY,
+	RSPAMD_RECEIVED_PART_FOR,
+	RSPAMD_RECEIVED_PART_WITH,
+	RSPAMD_RECEIVED_PART_ID,
+	RSPAMD_RECEIVED_PART_UNKNOWN,
+};
+
+struct received_part {
+	received_part_type type;
+	mime_string data;
+	std::vector<mime_string> comments;
+
+	explicit received_part(received_part_type t)
+		: type(t),
+		  data(received_char_filter)
+	{
+	}
+};
+
+static inline auto
+received_part_set_or_append(const gchar *begin,
+							gsize len,
+							mime_string &dest) -> void
+{
+	if (len == 0) {
+		return;
+	}
+
+	dest.append(begin, len);
+	dest.trim(" \t");
+}
+
+static auto
+received_process_part(const std::string_view &data,
+					  received_part_type type,
+					  std::ptrdiff_t &last,
+					  received_part &npart) -> bool
+{
+	auto obraces = 0, ebraces = 0;
+	auto seen_tcpinfo = false;
+	enum _parse_state {
+		skip_spaces,
+		in_comment,
+		read_data,
+		read_tcpinfo,
+		all_done
+	} state,
+		next_state;
+
+	/* In this function, we just process comments and data separately */
+	const auto *p = data.data();
+	const auto *end = p + data.size();
+	const auto *c = p;
+
+	state = skip_spaces;
+	next_state = read_data;
+
+	while (p < end) {
+		switch (state) {
+		case skip_spaces:
+			if (!g_ascii_isspace(*p)) {
+				c = p;
+				state = next_state;
+			}
+			else {
+				p++;
+			}
+			break;
+		case in_comment:
+			if (*p == '(') {
+				obraces++;
+			}
+			else if (*p == ')') {
+				ebraces++;
+
+				if (ebraces >= obraces) {
+					if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+						if (p > c) {
+							npart.comments.emplace_back(received_char_filter);
+							auto &comment = npart.comments.back();
+							received_part_set_or_append(c, p - c,
+														comment);
+						}
+					}
+
+					p++;
+					c = p;
+					state = skip_spaces;
+					next_state = read_data;
+
+					continue;
+				}
+			}
+
+			p++;
+			break;
+		case read_data:
+			if (*p == '(') {
+				if (p > c) {
+					if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+						received_part_set_or_append(c, p - c,
+													npart.data);
+					}
+				}
+
+				state = in_comment;
+				obraces = 1;
+				ebraces = 0;
+				p++;
+				c = p;
+			}
+			else if (g_ascii_isspace(*p)) {
+				if (p > c) {
+					if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+						received_part_set_or_append(c, p - c,
+													npart.data);
+					}
+				}
+
+				state = skip_spaces;
+				next_state = read_data;
+				c = p;
+			}
+			else if (*p == ';') {
+				/* It is actually delimiter of date part if not in the comments */
+				if (p > c) {
+					if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+						received_part_set_or_append(c, p - c,
+													npart.data);
+					}
+				}
+
+				state = all_done;
+				continue;
+			}
+			else if (npart.data.size() > 0) {
+				/* We have already received data and find something with no ( */
+				if (!seen_tcpinfo && type == received_part_type::RSPAMD_RECEIVED_PART_FROM) {
+					/* Check if we have something special here, such as TCPinfo */
+					if (*c == '[') {
+						state = read_tcpinfo;
+						p++;
+					}
+					else {
+						state = all_done;
+						continue;
+					}
+				}
+				else {
+					state = all_done;
+					continue;
+				}
+			}
+			else {
+				p++;
+			}
+			break;
+		case read_tcpinfo:
+			if (*p == ']') {
+				received_part_set_or_append(c, p - c + 1,
+											npart.data);
+				seen_tcpinfo = TRUE;
+				state = skip_spaces;
+				next_state = read_data;
+				c = p;
+			}
+			p++;
+			break;
+		case all_done:
+			if (p > data.data()) {
+				last = p - data.data();
+				return true;
+			}
+			else {
+				/* Empty element */
+				return false;
+			}
+			break;
+		}
+	}
+
+	/* Leftover */
+	switch (state) {
+	case read_data:
+		if (p > c) {
+			if (type != received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN) {
+				received_part_set_or_append(c, p - c,
+											npart.data);
+			}
+
+			last = p - data.data();
+
+			return true;
+		}
+		break;
+	case skip_spaces:
+		if (p > data.data()) {
+			last = p - data.data();
+
+			return true;
+		}
+	default:
+		break;
+	}
+
+	return false;
+}
+
+template<std::size_t N>
+constexpr auto lit_compare_lowercase(const char lit[N], const char *in) -> bool
+{
+	for (auto i = 0; i < N; i++) {
+		if (lc_map[(unsigned char) in[i]] != lit[i]) {
+			return false;
+		}
+	}
+
+	return true;
+}
+
+static auto
+received_spill(const std::string_view &in,
+			   std::ptrdiff_t &date_pos) -> std::vector<received_part>
+{
+	std::vector<received_part> parts;
+	std::ptrdiff_t pos = 0;
+	auto seen_from = false, seen_by = false;
+
+	const auto *p = in.data();
+	const auto *end = p + in.size();
+
+	auto skip_spaces = [&p, end]() {
+		while (p < end && g_ascii_isspace(*p)) {
+			p++;
+		}
+	};
+
+	skip_spaces();
+
+	/* Skip SMTP comments */
+	if (*p == '(') {
+		auto obraces = 0, ebraces = 0;
+
+		while (p < end) {
+			if (*p == ')') {
+				ebraces++;
+			}
+			else if (*p == '(') {
+				obraces++;
+			}
+
+			p++;
+
+			if (obraces == ebraces) {
+				/* Skip spaces after  */
+				skip_spaces();
+				break;
+			}
+		}
+	}
+
+	auto len = end - p;
+
+	if (len == 0) {
+		return parts;
+	}
+
+	auto maybe_process_part = [&](received_part_type what) -> bool {
+		parts.emplace_back(what);
+		auto &rcvd_part = parts.back();
+		auto chunk = std::string_view{p, (std::size_t)(end - p)};
+
+		if (!received_process_part(chunk, what, pos, rcvd_part)) {
+			parts.pop_back();
+
+			return false;
+		}
+
+		return true;
+	};
+
+	if (len > 4 && lit_compare_lowercase<4>("from", p)) {
+		p += sizeof("from") - 1;
+
+		/* We can now store from part */
+		if (!maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_FROM)) {
+			/* Do not accept malformed from */
+			return {};
+		}
+
+		g_assert(pos != 0);
+		p += pos;
+		len = end > p ? end - p : 0;
+		seen_from = true;
+	}
+
+	if (len > 2 && lit_compare_lowercase<2>("by", p)) {
+		p += sizeof("by") - 1;
+
+		if (!maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_BY)) {
+			return {};
+		}
+
+		g_assert(pos != 0);
+		p += pos;
+		len = end > p ? end - p : 0;
+		seen_by = true;
+	}
+
+	if (!seen_from && !seen_by) {
+		/* Useless received */
+		return {};
+	}
+
+	while (p < end) {
+		bool got_part = false;
+		if (*p == ';') {
+			/* We are at the date separator, stop here */
+			date_pos = p - in.data() + 1;
+			break;
+		}
+		else {
+			if (len > sizeof("with") && lit_compare_lowercase<4>("with", p)) {
+				p += sizeof("with") - 1;
+
+				got_part = maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_WITH);
+			}
+			else if (len > sizeof("for") && lit_compare_lowercase<3>("for", p)) {
+				p += sizeof("for") - 1;
+				got_part = maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_FOR);
+			}
+			else if (len > sizeof("id") && lit_compare_lowercase<2>("id", p)) {
+				p += sizeof("id") - 1;
+				got_part = maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_ID);
+			}
+			else {
+				while (p < end) {
+					if (!(g_ascii_isspace(*p) || *p == '(' || *p == ';')) {
+						p++;
+					}
+					else {
+						break;
+					}
+				}
+
+				if (p == end) {
+					return {};
+				}
+				else if (*p == ';') {
+					date_pos = p - in.data() + 1;
+					break;
+				}
+				else {
+					got_part = maybe_process_part(received_part_type::RSPAMD_RECEIVED_PART_UNKNOWN);
+				}
+			}
+
+			if (!got_part) {
+				p++;
+				len = end > p ? end - p : 0;
+			}
+			else {
+				g_assert(pos != 0);
+				p += pos;
+				len = end > p ? end - p : 0;
+			}
+		}
+	}
+
+	return parts;
+}
+
+#define RSPAMD_INET_ADDRESS_PARSE_RECEIVED \
+	(rspamd_inet_address_parse_flags)(RSPAMD_INET_ADDRESS_PARSE_REMOTE | RSPAMD_INET_ADDRESS_PARSE_NO_UNIX)
+
+static auto
+received_process_rdns(rspamd_mempool_t *pool,
+					  const std::string_view &in,
+					  mime_string &dest) -> bool
+{
+	auto seen_dot = false;
+
+	const auto *p = in.data();
+	const auto *end = p + in.size();
+
+	if (in.empty()) {
+		return false;
+	}
+
+	if (*p == '[' && *(end - 1) == ']' && in.size() > 2) {
+		/* We have enclosed ip address */
+		auto *addr = rspamd_parse_inet_address_pool(p + 1,
+													(end - p) - 2,
+													pool,
+													RSPAMD_INET_ADDRESS_PARSE_RECEIVED);
+
+		if (addr) {
+			const gchar *addr_str;
+
+			if (rspamd_inet_address_get_port(addr) != 0) {
+				addr_str = rspamd_inet_address_to_string_pretty(addr);
+			}
+			else {
+				addr_str = rspamd_inet_address_to_string(addr);
+			}
+
+			dest.assign_copy(std::string_view{addr_str});
+
+			return true;
+		}
+	}
+
+	auto hlen = 0u;
+
+	while (p < end) {
+		if (!g_ascii_isspace(*p) && rspamd_url_is_domain(*p)) {
+			if (*p == '.') {
+				seen_dot = true;
+			}
+
+			hlen++;
+		}
+		else {
+			break;
+		}
+
+		p++;
+	}
+
+	if (hlen > 0) {
+		if (p == end || (seen_dot && (g_ascii_isspace(*p) || *p == '[' || *p == '('))) {
+			/* All data looks like a hostname */
+			dest.assign_copy(std::string_view{in.data(), hlen});
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+static auto
+received_process_host_tcpinfo(rspamd_mempool_t *pool,
+							  received_header &rh,
+							  const std::string_view &in) -> bool
+{
+	rspamd_inet_addr_t *addr = nullptr;
+	auto ret = false;
+
+	if (in.empty()) {
+		return false;
+	}
+
+	if (in[0] == '[') {
+		/* Likely Exim version */
+
+		auto brace_pos = in.find(']');
+
+		if (brace_pos != std::string_view::npos) {
+			auto substr_addr = in.substr(1, brace_pos - 1);
+			addr = rspamd_parse_inet_address_pool(substr_addr.data(),
+												  substr_addr.size(),
+												  pool,
+												  RSPAMD_INET_ADDRESS_PARSE_RECEIVED);
+
+			if (addr) {
+				rh.addr = addr;
+				rh.real_ip.assign_copy(std::string_view(rspamd_inet_address_to_string(addr)));
+			}
+		}
+	}
+	else {
+		if (g_ascii_isxdigit(in[0])) {
+			/* Try to parse IP address */
+			addr = rspamd_parse_inet_address_pool(in.data(),
+												  in.size(), pool, RSPAMD_INET_ADDRESS_PARSE_RECEIVED);
+			if (addr) {
+				rh.addr = addr;
+				rh.real_ip.assign_copy(std::string_view(rspamd_inet_address_to_string(addr)));
+			}
+		}
+
+		if (!addr) {
+			/* Try canonical Postfix version: rdns [ip] */
+			auto obrace_pos = in.find('[');
+
+			if (obrace_pos != std::string_view::npos) {
+				auto ebrace_pos = in.rfind(']');
+
+				if (ebrace_pos != std::string_view::npos && ebrace_pos > obrace_pos) {
+					auto substr_addr = in.substr(obrace_pos + 1,
+												 ebrace_pos - obrace_pos - 1);
+					addr = rspamd_parse_inet_address_pool(substr_addr.data(),
+														  substr_addr.size(),
+														  pool,
+														  RSPAMD_INET_ADDRESS_PARSE_RECEIVED);
+
+					if (addr) {
+						rh.addr = addr;
+						rh.real_ip.assign_copy(std::string_view(rspamd_inet_address_to_string(addr)));
+
+						/* Process with rDNS */
+						auto rdns_substr = in.substr(0, obrace_pos);
+
+						if (received_process_rdns(pool, rdns_substr, rh.real_hostname)) {
+							ret = true;
+						}
+					}
+				}
+			}
+			else {
+				/* Hostname or some crap, sigh... */
+				if (received_process_rdns(pool, in, rh.real_hostname)) {
+					ret = true;
+				}
+			}
+		}
+	}
+
+	return ret;
+}
+
+static void
+received_process_from(rspamd_mempool_t *pool,
+					  const received_part &rpart,
+					  received_header &rh)
+{
+	if (rpart.data.size() > 0) {
+		/* We have seen multiple cases:
+		 * - [ip] (hostname/unknown [real_ip])
+		 * - helo (hostname/unknown [real_ip])
+		 * - [ip]
+		 * - hostname
+		 * - hostname ([ip]:port helo=xxx)
+		 * Maybe more...
+		 */
+		auto seen_ip_in_data = false;
+
+		if (!rpart.comments.empty()) {
+			/* We can have info within comment as part of RFC */
+			received_process_host_tcpinfo(
+				pool, rh,
+				rpart.comments[0].as_view());
+		}
+
+		if (rh.real_ip.size() == 0) {
+			/* Try to do the same with data */
+			if (received_process_host_tcpinfo(
+					pool, rh,
+					rpart.data.as_view())) {
+				seen_ip_in_data = true;
+			}
+		}
+
+		if (!seen_ip_in_data) {
+			if (rh.real_ip.size() != 0) {
+				/* Get announced hostname (usually helo) */
+				received_process_rdns(pool,
+									  rpart.data.as_view(),
+									  rh.from_hostname);
+			}
+			else {
+				received_process_host_tcpinfo(pool,
+											  rh, rpart.data.as_view());
+			}
+		}
+	}
+	else {
+		/* rpart->dlen = 0 */
+		if (!rpart.comments.empty()) {
+			received_process_host_tcpinfo(
+				pool, rh,
+				rpart.comments[0].as_view());
+		}
+	}
+}
+
+static auto
+received_header_parse(received_header_chain &chain, rspamd_mempool_t *pool,
+					  const std::string_view &in,
+					  struct rspamd_mime_header *hdr) -> bool
+{
+	std::ptrdiff_t date_pos = -1;
+
+	static constexpr const auto protos_map = frozen::make_unordered_map<frozen::string, received_flags>({{"smtp", received_flags::SMTP},
+																										 {"esmtp", received_flags::ESMTP},
+																										 {"esmtpa", received_flags::ESMTPA |
+																														received_flags::AUTHENTICATED},
+																										 {"esmtpsa", received_flags::ESMTPSA |
+																														 received_flags::SSL |
+																														 received_flags::AUTHENTICATED},
+																										 {"esmtps", received_flags::ESMTPS |
+																														received_flags::SSL},
+																										 {"lmtp", received_flags::LMTP},
+																										 {"imap", received_flags::IMAP},
+																										 {"imaps", received_flags::IMAP |
+																													   received_flags::SSL},
+																										 {"http", received_flags::HTTP},
+																										 {"https", received_flags::HTTP |
+																													   received_flags::SSL},
+																										 {"local", received_flags::LOCAL}});
+
+	auto parts = received_spill(in, date_pos);
+
+	if (parts.empty()) {
+		return false;
+	}
+
+	auto &rh = chain.new_received();
+
+	rh.flags = received_flags::UNKNOWN;
+	rh.hdr = hdr;
+
+	for (const auto &part: parts) {
+		switch (part.type) {
+		case received_part_type::RSPAMD_RECEIVED_PART_FROM:
+			received_process_from(pool, part, rh);
+			break;
+		case received_part_type::RSPAMD_RECEIVED_PART_BY:
+			received_process_rdns(pool,
+								  part.data.as_view(),
+								  rh.by_hostname);
+			break;
+		case received_part_type::RSPAMD_RECEIVED_PART_WITH:
+			if (part.data.size() > 0) {
+				auto proto_flag_it = protos_map.find(part.data.as_view());
+
+				if (proto_flag_it != protos_map.end()) {
+					rh.flags = proto_flag_it->second;
+				}
+			}
+			break;
+		case received_part_type::RSPAMD_RECEIVED_PART_FOR:
+			rh.for_mbox.assign_copy(part.data);
+			rh.for_addr = rspamd_email_address_from_smtp(rh.for_mbox.data(),
+														 rh.for_mbox.size());
+			break;
+		default:
+			/* Do nothing */
+			break;
+		}
+	}
+
+	if (!rh.real_hostname.empty() && rh.from_hostname.empty()) {
+		rh.from_hostname.assign_copy(rh.real_hostname);
+	}
+
+	if (date_pos > 0 && date_pos < in.size()) {
+		auto date_sub = in.substr(date_pos);
+		rh.timestamp = rspamd_parse_smtp_date((const unsigned char *) date_sub.data(),
+											  date_sub.size(), nullptr);
+	}
+
+	return true;
+}
+
+static auto
+received_maybe_fix_task(struct rspamd_task *task) -> bool
+{
+	auto *recv_chain_ptr = static_cast<received_header_chain *>(MESSAGE_FIELD(task, received_headers));
+
+	if (recv_chain_ptr) {
+		auto need_recv_correction = false;
+
+		auto top_recv_maybe = recv_chain_ptr->get_received(0);
+
+		if (top_recv_maybe.has_value()) {
+			auto &top_recv = top_recv_maybe.value().get();
+
+			const auto *raddr = top_recv.addr;
+			if (top_recv.real_ip.size() == 0 || (task->cfg && task->cfg->ignore_received)) {
+				need_recv_correction = true;
+			}
+			else if (!(task->flags & RSPAMD_TASK_FLAG_NO_IP) && task->from_addr) {
+				if (!raddr) {
+					need_recv_correction = true;
+				}
+				else {
+					if (rspamd_inet_address_compare(raddr, task->from_addr, FALSE) != 0) {
+						need_recv_correction = true;
+					}
+				}
+			}
+
+			if (need_recv_correction && !(task->flags & RSPAMD_TASK_FLAG_NO_IP) && task->from_addr) {
+				msg_debug_task("the first received seems to be"
+							   " not ours, prepend it with fake one");
+
+				auto &trecv = recv_chain_ptr->new_received(received_header_chain::append_type::append_head);
+				trecv.flags |= received_flags::ARTIFICIAL;
+
+				if (task->flags & RSPAMD_TASK_FLAG_SSL) {
+					trecv.flags |= received_flags::SSL;
+				}
+
+				if (task->auth_user) {
+					trecv.flags |= received_flags::AUTHENTICATED;
+				}
+
+				trecv.real_ip.assign_copy(std::string_view(rspamd_inet_address_to_string(task->from_addr)));
+
+				const auto *mta_name = (const char *) rspamd_mempool_get_variable(task->task_pool,
+																				  RSPAMD_MEMPOOL_MTA_NAME);
+
+				if (mta_name) {
+					trecv.by_hostname.assign_copy(std::string_view(mta_name));
+				}
+				trecv.addr = rspamd_inet_address_copy(task->from_addr,
+													  task->task_pool);
+
+				if (task->hostname) {
+					trecv.real_hostname.assign_copy(std::string_view(task->hostname));
+					trecv.from_hostname.assign_copy(trecv.real_hostname);
+				}
+
+				return true;
+			}
+
+			/* Extract data from received header if we were not given IP */
+			if (!need_recv_correction && (task->flags & RSPAMD_TASK_FLAG_NO_IP) &&
+				(task->cfg && !task->cfg->ignore_received)) {
+				if (!top_recv.real_ip.empty()) {
+					if (!rspamd_parse_inet_address(&task->from_addr,
+												   top_recv.real_ip.data(),
+												   top_recv.real_ip.size(),
+												   RSPAMD_INET_ADDRESS_PARSE_NO_UNIX)) {
+						msg_warn_task("cannot get IP from received header: '%s'",
+									  top_recv.real_ip.data());
+						task->from_addr = nullptr;
+					}
+				}
+				if (!top_recv.real_hostname.empty()) {
+					task->hostname = top_recv.real_hostname.data();
+				}
+
+				return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+static auto
+received_export_to_lua(received_header_chain *chain, lua_State *L) -> bool
+{
+	if (chain == nullptr) {
+		return false;
+	}
+
+	lua_createtable(L, chain->size(), 0);
+
+	auto push_flag = [L](const received_header &rh, received_flags fl, const char *name) {
+		lua_pushboolean(L, !!(rh.flags & fl));
+		lua_setfield(L, -2, name);
+	};
+
+	auto i = 1;
+
+	for (const auto &rh: chain->as_vector()) {
+		lua_createtable(L, 0, 10);
+
+		if (rh.hdr && rh.hdr->decoded) {
+			rspamd_lua_table_set(L, "raw", rh.hdr->decoded);
+		}
+
+		lua_createtable(L, 0, 3);
+		push_flag(rh, received_flags::ARTIFICIAL, "artificial");
+		push_flag(rh, received_flags::AUTHENTICATED, "authenticated");
+		push_flag(rh, received_flags::SSL, "ssl");
+		lua_setfield(L, -2, "flags");
+
+		auto push_nullable_string = [L](const mime_string &st, const char *field) {
+			if (st.empty()) {
+				lua_pushnil(L);
+			}
+			else {
+				lua_pushlstring(L, st.data(), st.size());
+			}
+			lua_setfield(L, -2, field);
+		};
+
+		push_nullable_string(rh.from_hostname, "from_hostname");
+		push_nullable_string(rh.real_hostname, "real_hostname");
+		push_nullable_string(rh.real_ip, "from_ip");
+		push_nullable_string(rh.by_hostname, "by_hostname");
+		push_nullable_string(rh.for_mbox, "for");
+
+		if (rh.addr) {
+			rspamd_lua_ip_push(L, rh.addr);
+		}
+		else {
+			lua_pushnil(L);
+		}
+		lua_setfield(L, -2, "real_ip");
+
+		lua_pushstring(L, received_protocol_to_string(rh.flags));
+		lua_setfield(L, -2, "proto");
+
+		lua_pushinteger(L, rh.timestamp);
+		lua_setfield(L, -2, "timestamp");
+
+		lua_rawseti(L, -2, i++);
+	}
+
+	return true;
+}
+
+}// namespace rspamd::mime
+
+bool rspamd_received_header_parse(struct rspamd_task *task,
+								  const char *data, size_t sz,
+								  struct rspamd_mime_header *hdr)
+{
+	auto *recv_chain_ptr = static_cast<rspamd::mime::received_header_chain *>(MESSAGE_FIELD(task, received_headers));
+
+	if (recv_chain_ptr == nullptr) {
+		/* This constructor automatically registers dtor in mempool */
+		recv_chain_ptr = new rspamd::mime::received_header_chain(task);
+		MESSAGE_FIELD(task, received_headers) = (void *) recv_chain_ptr;
+	}
+	return rspamd::mime::received_header_parse(*recv_chain_ptr, task->task_pool,
+											   std::string_view{data, sz}, hdr);
+}
+
+bool rspamd_received_maybe_fix_task(struct rspamd_task *task)
+{
+	return rspamd::mime::received_maybe_fix_task(task);
+}
+
+bool rspamd_received_export_to_lua(struct rspamd_task *task, lua_State *L)
+{
+	return rspamd::mime::received_export_to_lua(
+		static_cast<rspamd::mime::received_header_chain *>(MESSAGE_FIELD(task, received_headers)),
+		L);
+}
+
+/* Tests part */
+#define DOCTEST_CONFIG_IMPLEMENTATION_IN_DLL
+#include "doctest/doctest.h"
+
+TEST_SUITE("received")
+{
+	TEST_CASE("parse received")
+	{
+		using namespace std::string_view_literals;
+		using map_type = ankerl::unordered_dense::map<std::string_view, std::string_view>;
+		std::vector<std::pair<std::string_view, map_type>> cases{
+			// Simple received
+			{"from smtp11.mailtrack.pl (smtp11.mailtrack.pl [185.243.30.90])"sv,
+			 {{"real_ip", "185.243.30.90"},
+			  {"real_hostname", "smtp11.mailtrack.pl"},
+			  {"from_hostname", "smtp11.mailtrack.pl"}}},
+			// Real Postfix IPv6 received
+			{"from server.chat-met-vreemden.nl (unknown [IPv6:2a01:7c8:aab6:26d:5054:ff:fed1:1da2])\n"
+			 "\t(using TLSv1.2 with cipher ECDHE-RSA-AES256-GCM-SHA384 (256/256 bits))\n"
+			 "\t(Client did not present a certificate)\n"
+			 "\tby mx1.freebsd.org (Postfix) with ESMTPS id CF0171862\n"
+			 "\tfor <test@example.com>; Mon,  6 Jul 2015 09:01:20 +0000 (UTC)\n"
+			 "\t(envelope-from upwest201diana@outlook.com)"sv,
+			 {{"real_ip", "2a01:7c8:aab6:26d:5054:ff:fed1:1da2"},
+			  {"from_hostname", "server.chat-met-vreemden.nl"},
+			  {"by_hostname", "mx1.freebsd.org"},
+			  {"for_mbox", "<test@example.com>"}}},
+			// Exim IPv4 received
+			{"from localhost ([127.0.0.1]:49019 helo=hummus.csx.cam.ac.uk)\n"
+			 " by hummus.csx.cam.ac.uk with esmtp (Exim 4.91-pdpfix1)\n"
+			 " (envelope-from <exim-dev-bounces@exim.org>)\n"
+			 " id 1fZ55o-0006DP-3H\n"
+			 " for <xxx@xxx.xxx>; Sat, 30 Jun 2018 02:54:28 +0100"sv,
+			 {
+				 {"from_hostname", "localhost"},
+				 {"real_ip", "127.0.0.1"},
+				 {"for_mbox", "<xxx@xxx.xxx>"},
+				 {"by_hostname", "hummus.csx.cam.ac.uk"},
+			 }},
+			// Exim IPv6 received
+			{"from smtp.spodhuis.org ([2a02:898:31:0:48:4558:736d:7470]:38689\n"
+			 " helo=mx.spodhuis.org)\n"
+			 " by hummus.csx.cam.ac.uk with esmtpsa (TLSv1.3:TLS_AES_256_GCM_SHA384:256)\n"
+			 " (Exim 4.91-pdpfix1+cc) (envelope-from <xxx@exim.org>)\n"
+			 " id 1fZ55k-0006CO-9M\n"
+			 " for exim-dev@exim.org; Sat, 30 Jun 2018 02:54:24 +0100"sv,
+			 {
+				 {"from_hostname", "smtp.spodhuis.org"},
+				 {"real_ip", "2a02:898:31:0:48:4558:736d:7470"},
+				 {"for_mbox", "exim-dev@exim.org"},
+				 {"by_hostname", "hummus.csx.cam.ac.uk"},
+			 }},
+			// Haraka received
+			{"from aaa.cn ([1.1.1.1]) by localhost.localdomain (Haraka/2.8.18) with "
+			 "ESMTPA id 349C9C2B-491A-4925-A687-3EF14038C344.1 envelope-from <huxin@xxx.com> "
+			 "(authenticated bits=0); Tue, 03 Jul 2018 14:18:13 +0200"sv,
+			 {
+				 {"from_hostname", "aaa.cn"},
+				 {"real_ip", "1.1.1.1"},
+				 {"by_hostname", "localhost.localdomain"},
+			 }},
+			// Invalid by
+			{"from [192.83.172.101] (HELLO 148.251.238.35) (148.251.238.35) "
+			 "by guovswzqkvry051@sohu.com with gg login "
+			 "by AOL 6.0 for Windows US sub 008 SMTP  ; Tue, 03 Jul 2018 09:01:47 -0300"sv,
+			 {
+				 {"from_hostname", "192.83.172.101"},
+				 {"real_ip", "192.83.172.101"},
+			 }},
+			// Invalid hostinfo
+			{"from example.com ([]) by example.com with ESMTP id 2019091111 ;"
+			 " Thu, 26 Sep 2019 11:19:07 +0200"sv,
+			 {
+				 {"by_hostname", "example.com"},
+				 {"from_hostname", "example.com"},
+				 {"real_hostname", "example.com"},
+			 }},
+			// Different real and announced hostnames + broken crap
+			{"from 171-29.br (1-1-1-1.z.com.br [1.1.1.1]) by x.com.br (Postfix) "
+			 "with;ESMTP id 44QShF6xj4z1X for <hey@y.br>; Thu, 21 Mar 2019 23:45:46 -0300 "
+			 ": <g @yi.br>"sv,
+			 {
+				 {"real_ip", "1.1.1.1"},
+				 {"from_hostname", "171-29.br"},
+				 {"real_hostname", "1-1-1-1.z.com.br"},
+				 {"by_hostname", "x.com.br"},
+			 }},
+			// Different real and announced ips + no hostname
+			{"from [127.0.0.1] ([127.0.0.2]) by smtp.gmail.com with ESMTPSA id xxxololo"sv,
+			 {
+				 {"real_ip", "127.0.0.2"},
+				 {"from_hostname", "127.0.0.1"},
+				 {"by_hostname", "smtp.gmail.com"},
+			 }},
+			// Different real and hostanes
+			{"from 185.118.166.127 (steven2.zhou01.pserver.ru [185.118.166.127]) "
+			 "by mail.832zsu.cn (Postfix) with ESMTPA id AAD722133E34"sv,
+			 {
+				 {"real_ip", "185.118.166.127"},
+				 {"from_hostname", "185.118.166.127"},
+				 {"real_hostname", "steven2.zhou01.pserver.ru"},
+				 {"by_hostname", "mail.832zsu.cn"},
+			 }},
+			// \0 in received must be filtered
+			{"from smtp11.mailt\0rack.pl (smtp11.mail\0track.pl [1\085.243.30.90])"sv,
+			 {{"real_ip", "185.243.30.90"},
+			  {"real_hostname", "smtp11.mailtrack.pl"},
+			  {"from_hostname", "smtp11.mailtrack.pl"}}},
+			// No from part
+			{"by mail.832zsu.cn (Postfix) with ESMTPA id AAD722133E34"sv,
+			 {
+				 {"by_hostname", "mail.832zsu.cn"},
+			 }},
+			// From part is in the comment
+			{"(from asterisk@localhost)\n"
+			 "        by pbx.xxx.com (8.14.7/8.14.7/Submit) id 076Go4wD014562;\n"
+			 "        Thu, 6 Aug 2020 11:50:04 -0500"sv,
+			 {
+				 {"by_hostname", "pbx.xxx.com"},
+			 }},
+		};
+		rspamd_mempool_t *pool = rspamd_mempool_new_default("rcvd test", 0);
+
+		for (auto &&c: cases) {
+			SUBCASE(c.first.data())
+			{
+				rspamd::mime::received_header_chain chain;
+				auto ret = rspamd::mime::received_header_parse(chain, pool,
+															   c.first, nullptr);
+				CHECK(ret == true);
+				auto &&rh = chain.get_received(0);
+				CHECK(rh.has_value());
+				auto res = rh.value().get().as_map();
+
+				for (const auto &expected: c.second) {
+					CHECK_MESSAGE(res.contains(expected.first), expected.first.data());
+					CHECK(res[expected.first] == expected.second);
+				}
+				for (const auto &existing: res) {
+					CHECK_MESSAGE(c.second.contains(existing.first), existing.first.data());
+					CHECK(c.second[existing.first] == existing.second);
+				}
+			}
+		}
+
+		rspamd_mempool_delete(pool);
+	}
+}
\ No newline at end of file
diff --git a/src/libmime/received.h b/src/libmime/received.h
new file mode 100644
index 0000000..46608a3
--- /dev/null
+++ b/src/libmime/received.h
@@ -0,0 +1,68 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef RSPAMD_RECEIVED_H
+#define RSPAMD_RECEIVED_H
+
+#include "config.h"
+#include "libutil/addr.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+/*
+ * C bindings for C++ received code
+ */
+
+struct rspamd_email_address;
+struct rspamd_received_header_chain;
+struct rspamd_mime_header;
+
+/**
+ * Parse received header from an input header data
+ * @param task
+ * @param data
+ * @param sz
+ * @param hdr
+ * @return
+ */
+bool rspamd_received_header_parse(struct rspamd_task *task,
+								  const char *data, size_t sz, struct rspamd_mime_header *hdr);
+
+
+/**
+ * Process task data and the most top received and fix either part if needed
+ * @param task
+ * @return
+ */
+bool rspamd_received_maybe_fix_task(struct rspamd_task *task);
+
+struct lua_State;
+/**
+ * Push received headers chain to lua
+ * @param task
+ * @param L
+ * @return
+ */
+bool rspamd_received_export_to_lua(struct rspamd_task *task, struct lua_State *L);
+
+#ifdef __cplusplus
+}
+#endif
+
+
+#endif//RSPAMD_RECEIVED_H
diff --git a/src/libmime/received.hxx b/src/libmime/received.hxx
new file mode 100644
index 0000000..4f423f1
--- /dev/null
+++ b/src/libmime/received.hxx
@@ -0,0 +1,314 @@
+/*-
+ * Copyright 2021 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+
+#ifndef RSPAMD_RECEIVED_HXX
+#define RSPAMD_RECEIVED_HXX
+#pragma once
+
+#include "config.h"
+#include "received.h"
+#include "mime_string.hxx"
+#include "libmime/email_addr.h"
+#include "libserver/task.h"
+#include "contrib/ankerl/unordered_dense.h"
+#include <vector>
+#include <string_view>
+#include <utility>
+#include <optional>
+
+namespace rspamd::mime {
+
+static inline auto
+received_char_filter(UChar32 uc) -> UChar32
+{
+	if (u_isprint(uc)) {
+		return u_tolower(uc);
+	}
+
+	return 0;
+}
+
+enum class received_flags {
+	DEFAULT = 0,
+	SMTP = 1u << 0u,
+	ESMTP = 1u << 1u,
+	ESMTPA = 1u << 2u,
+	ESMTPS = 1u << 3u,
+	ESMTPSA = 1u << 4u,
+	LMTP = 1u << 5u,
+	IMAP = 1u << 6u,
+	LOCAL = 1u << 7u,
+	HTTP = 1u << 8u,
+	MAPI = 1u << 9u,
+	UNKNOWN = 1u << 10u,
+	ARTIFICIAL = (1u << 11u),
+	SSL = (1u << 12u),
+	AUTHENTICATED = (1u << 13u),
+};
+
+constexpr received_flags operator|(received_flags lhs, received_flags rhs)
+{
+	using ut = std::underlying_type<received_flags>::type;
+	return static_cast<received_flags>(static_cast<ut>(lhs) | static_cast<ut>(rhs));
+}
+
+constexpr received_flags operator|=(received_flags &lhs, const received_flags rhs)
+{
+	using ut = std::underlying_type<received_flags>::type;
+	lhs = static_cast<received_flags>(static_cast<ut>(lhs) | static_cast<ut>(rhs));
+	return lhs;
+}
+
+constexpr received_flags operator&(received_flags lhs, received_flags rhs)
+{
+	using ut = std::underlying_type<received_flags>::type;
+	return static_cast<received_flags>(static_cast<ut>(lhs) & static_cast<ut>(rhs));
+}
+
+constexpr bool operator!(received_flags fl)
+{
+	return fl == received_flags::DEFAULT;
+}
+
+constexpr received_flags received_type_apply_protocols_mask(received_flags fl)
+{
+	return fl & (received_flags::SMTP |
+				 received_flags::ESMTP |
+				 received_flags::ESMTPA |
+				 received_flags::ESMTPS |
+				 received_flags::ESMTPSA |
+				 received_flags::IMAP |
+				 received_flags::HTTP |
+				 received_flags::LOCAL |
+				 received_flags::MAPI |
+				 received_flags::LMTP);
+}
+
+constexpr const char *received_protocol_to_string(received_flags fl)
+{
+	const auto *proto = "unknown";
+
+	switch (received_type_apply_protocols_mask(fl)) {
+	case received_flags::SMTP:
+		proto = "smtp";
+		break;
+	case received_flags::ESMTP:
+		proto = "esmtp";
+		break;
+	case received_flags::ESMTPS:
+		proto = "esmtps";
+		break;
+	case received_flags::ESMTPA:
+		proto = "esmtpa";
+		break;
+	case received_flags::ESMTPSA:
+		proto = "esmtpsa";
+		break;
+	case received_flags::LMTP:
+		proto = "lmtp";
+		break;
+	case received_flags::IMAP:
+		proto = "imap";
+		break;
+	case received_flags::HTTP:
+		proto = "http";
+		break;
+	case received_flags::LOCAL:
+		proto = "local";
+		break;
+	case received_flags::MAPI:
+		proto = "mapi";
+		break;
+	default:
+		break;
+	}
+
+	return proto;
+}
+
+struct received_header {
+	mime_string from_hostname;
+	mime_string real_hostname;
+	mime_string real_ip;
+	mime_string by_hostname;
+	mime_string for_mbox;
+	struct rspamd_email_address *for_addr = nullptr;
+	rspamd_inet_addr_t *addr = nullptr;
+	struct rspamd_mime_header *hdr = nullptr;
+	time_t timestamp = 0;
+	received_flags flags = received_flags::DEFAULT; /* See enum rspamd_received_type */
+
+	received_header() noexcept
+		: from_hostname(received_char_filter),
+		  real_hostname(received_char_filter),
+		  real_ip(received_char_filter),
+		  by_hostname(received_char_filter),
+		  for_mbox()
+	{
+	}
+	/* We have raw C pointers, so copy is explicitly disabled */
+	received_header(const received_header &other) = delete;
+	received_header(received_header &&other) noexcept
+	{
+		*this = std::move(other);
+	}
+
+	received_header &operator=(received_header &&other) noexcept
+	{
+		if (this != &other) {
+			from_hostname = std::move(other.from_hostname);
+			real_hostname = std::move(other.real_hostname);
+			real_ip = std::move(other.real_ip);
+			by_hostname = std::move(other.by_hostname);
+			for_mbox = std::move(other.for_mbox);
+			timestamp = other.timestamp;
+			flags = other.flags;
+			std::swap(for_addr, other.for_addr);
+			std::swap(addr, other.addr);
+			std::swap(hdr, other.hdr);
+		}
+		return *this;
+	}
+
+	/* Unit tests helper */
+	static auto from_map(const ankerl::unordered_dense::map<std::string_view, std::string_view> &map) -> received_header
+	{
+		using namespace std::string_view_literals;
+		received_header rh;
+
+		if (map.contains("from_hostname")) {
+			rh.from_hostname.assign_copy(map.at("from_hostname"sv));
+		}
+		if (map.contains("real_hostname")) {
+			rh.real_hostname.assign_copy(map.at("real_hostname"sv));
+		}
+		if (map.contains("by_hostname")) {
+			rh.by_hostname.assign_copy(map.at("by_hostname"sv));
+		}
+		if (map.contains("real_ip")) {
+			rh.real_ip.assign_copy(map.at("real_ip"sv));
+		}
+		if (map.contains("for_mbox")) {
+			rh.for_mbox.assign_copy(map.at("for_mbox"sv));
+		}
+
+		return rh;
+	}
+
+	auto as_map() const -> ankerl::unordered_dense::map<std::string_view, std::string_view>
+	{
+		ankerl::unordered_dense::map<std::string_view, std::string_view> map;
+
+		if (!from_hostname.empty()) {
+			map["from_hostname"] = from_hostname.as_view();
+		}
+		if (!real_hostname.empty()) {
+			map["real_hostname"] = real_hostname.as_view();
+		}
+		if (!by_hostname.empty()) {
+			map["by_hostname"] = by_hostname.as_view();
+		}
+		if (!real_ip.empty()) {
+			map["real_ip"] = real_ip.as_view();
+		}
+		if (!for_mbox.empty()) {
+			map["for_mbox"] = for_mbox.as_view();
+		}
+
+		return map;
+	}
+
+	~received_header()
+	{
+		if (for_addr) {
+			rspamd_email_address_free(for_addr);
+		}
+	}
+};
+
+class received_header_chain {
+public:
+	explicit received_header_chain(struct rspamd_task *task)
+	{
+		headers.reserve(2);
+		rspamd_mempool_add_destructor(task->task_pool,
+									  received_header_chain::received_header_chain_pool_dtor, this);
+	}
+	explicit received_header_chain()
+	{
+		headers.reserve(2);
+	}
+
+	enum class append_type {
+		append_tail,
+		append_head
+	};
+
+	auto new_received(append_type how = append_type::append_tail) -> received_header &
+	{
+		if (how == append_type::append_tail) {
+			headers.emplace_back();
+
+			return headers.back();
+		}
+		else {
+			headers.insert(std::begin(headers), received_header());
+
+			return headers.front();
+		}
+	}
+	auto new_received(received_header &&hdr, append_type how = append_type::append_tail) -> received_header &
+	{
+		if (how == append_type::append_tail) {
+			headers.emplace_back(std::move(hdr));
+
+			return headers.back();
+		}
+		else {
+			headers.insert(std::begin(headers), std::move(hdr));
+
+			return headers.front();
+		}
+	}
+	auto get_received(std::size_t nth) -> std::optional<std::reference_wrapper<received_header>>
+	{
+		if (nth < headers.size()) {
+			return headers[nth];
+		}
+
+		return std::nullopt;
+	}
+	auto size() const -> std::size_t
+	{
+		return headers.size();
+	}
+	constexpr auto as_vector() const -> const std::vector<received_header> &
+	{
+		return headers;
+	}
+
+private:
+	static auto received_header_chain_pool_dtor(void *ptr) -> void
+	{
+		delete static_cast<received_header_chain *>(ptr);
+	}
+	std::vector<received_header> headers;
+};
+
+}// namespace rspamd::mime
+
+#endif//RSPAMD_RECEIVED_HXX
diff --git a/src/libmime/scan_result.c b/src/libmime/scan_result.c
new file mode 100644
index 0000000..a6bc0cb
--- /dev/null
+++ b/src/libmime/scan_result.c
@@ -0,0 +1,1106 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "mem_pool.h"
+#include "scan_result.h"
+#include "rspamd.h"
+#include "message.h"
+#include "lua/lua_common.h"
+#include "libserver/cfg_file_private.h"
+#include "libmime/scan_result_private.h"
+#include "contrib/fastutf8/fastutf8.h"
+#include <math.h>
+#include "contrib/uthash/utlist.h"
+
+#define msg_debug_metric(...) rspamd_conditional_debug_fast(NULL, NULL,                                               \
+															rspamd_metric_log_id, "metric", task->task_pool->tag.uid, \
+															RSPAMD_LOG_FUNC,                                          \
+															__VA_ARGS__)
+
+INIT_LOG_MODULE(metric)
+
+/* Average symbols count to optimize hash allocation */
+static struct rspamd_counter_data symbols_count;
+
+static void
+rspamd_scan_result_dtor(gpointer d)
+{
+	struct rspamd_scan_result *r = (struct rspamd_scan_result *) d;
+	struct rspamd_symbol_result *sres;
+
+	rspamd_set_counter_ema(&symbols_count, kh_size(r->symbols), 0.5);
+
+	if (r->symbol_cbref != -1) {
+		luaL_unref(r->task->cfg->lua_state, LUA_REGISTRYINDEX, r->symbol_cbref);
+	}
+
+	kh_foreach_value(r->symbols, sres, {
+		if (sres->options) {
+			kh_destroy(rspamd_options_hash, sres->options);
+		}
+	});
+
+	kh_destroy(rspamd_symbols_hash, r->symbols);
+	kh_destroy(rspamd_symbols_group_hash, r->sym_groups);
+}
+
+static void
+rspamd_metric_actions_foreach_cb(int i, struct rspamd_action *act, void *cbd)
+{
+	struct rspamd_scan_result *metric_res = (struct rspamd_scan_result *) cbd;
+	metric_res->actions_config[i].flags = RSPAMD_ACTION_RESULT_DEFAULT;
+	if (!(act->flags & RSPAMD_ACTION_NO_THRESHOLD)) {
+		metric_res->actions_config[i].cur_limit = act->threshold;
+	}
+	else {
+		metric_res->actions_config[i].flags |= RSPAMD_ACTION_RESULT_NO_THRESHOLD;
+	}
+	metric_res->actions_config[i].action = act;
+}
+
+struct rspamd_scan_result *
+rspamd_create_metric_result(struct rspamd_task *task,
+							const gchar *name, gint lua_sym_cbref)
+{
+	struct rspamd_scan_result *metric_res;
+
+	metric_res = rspamd_mempool_alloc0(task->task_pool,
+									   sizeof(struct rspamd_scan_result));
+	metric_res->symbols = kh_init(rspamd_symbols_hash);
+	metric_res->sym_groups = kh_init(rspamd_symbols_group_hash);
+
+	if (name) {
+		metric_res->name = rspamd_mempool_strdup(task->task_pool, name);
+	}
+	else {
+		metric_res->name = NULL;
+	}
+
+	metric_res->symbol_cbref = lua_sym_cbref;
+	metric_res->task = task;
+
+	/* Optimize allocation */
+	kh_resize(rspamd_symbols_group_hash, metric_res->sym_groups, 4);
+
+	if (symbols_count.mean > 4) {
+		kh_resize(rspamd_symbols_hash, metric_res->symbols, symbols_count.mean);
+	}
+	else {
+		kh_resize(rspamd_symbols_hash, metric_res->symbols, 4);
+	}
+
+	if (task->cfg) {
+		size_t nact = rspamd_config_actions_size(task->cfg);
+		metric_res->actions_config = rspamd_mempool_alloc0(task->task_pool,
+														   sizeof(struct rspamd_action_config) * nact);
+		rspamd_config_actions_foreach_enumerate(task->cfg, rspamd_metric_actions_foreach_cb, metric_res);
+		metric_res->nactions = nact;
+	}
+
+	rspamd_mempool_add_destructor(task->task_pool,
+								  rspamd_scan_result_dtor,
+								  metric_res);
+	DL_APPEND(task->result, metric_res);
+
+	return metric_res;
+}
+
+static inline int
+rspamd_pr_sort(const struct rspamd_passthrough_result *pra,
+			   const struct rspamd_passthrough_result *prb)
+{
+	return prb->priority - pra->priority;
+}
+
+bool rspamd_add_passthrough_result(struct rspamd_task *task,
+								   struct rspamd_action *action,
+								   guint priority,
+								   double target_score,
+								   const gchar *message,
+								   const gchar *module,
+								   uint flags,
+								   struct rspamd_scan_result *scan_result)
+{
+	struct rspamd_passthrough_result *pr;
+
+	if (scan_result == NULL) {
+		scan_result = task->result;
+	}
+
+	/* Find the specific action config */
+	struct rspamd_action_config *action_config = NULL;
+
+	for (unsigned int i = 0; i < scan_result->nactions; i++) {
+		struct rspamd_action_config *cur = &scan_result->actions_config[i];
+
+		/* We assume that all action pointers are static */
+		if (cur->action == action) {
+			action_config = cur;
+			break;
+		}
+	}
+
+	if (action_config && (action_config->flags & RSPAMD_ACTION_RESULT_DISABLED)) {
+		msg_info_task("<%s>: NOT set pre-result to '%s' %s(%.2f): '%s' from %s(%d); action is disabled",
+					  MESSAGE_FIELD_CHECK(task, message_id), action->name,
+					  flags & RSPAMD_PASSTHROUGH_LEAST ? "*least " : "",
+					  target_score,
+					  message, module, priority);
+
+		return false;
+	}
+
+	pr = rspamd_mempool_alloc(task->task_pool, sizeof(*pr));
+	pr->action = action;
+	pr->priority = priority;
+	pr->message = message;
+	pr->module = module;
+	pr->target_score = target_score;
+	pr->flags = flags;
+
+	DL_APPEND(scan_result->passthrough_result, pr);
+	DL_SORT(scan_result->passthrough_result, rspamd_pr_sort);
+
+	if (!isnan(target_score)) {
+
+		msg_info_task("<%s>: set pre-result to '%s' %s(%.2f): '%s' from %s(%d)",
+					  MESSAGE_FIELD_CHECK(task, message_id), action->name,
+					  flags & RSPAMD_PASSTHROUGH_LEAST ? "*least " : "",
+					  target_score,
+					  message, module, priority);
+	}
+	else {
+		msg_info_task("<%s>: set pre-result to '%s' %s(no score): '%s' from %s(%d)",
+					  MESSAGE_FIELD_CHECK(task, message_id), action->name,
+					  flags & RSPAMD_PASSTHROUGH_LEAST ? "*least " : "",
+					  message, module, priority);
+	}
+
+	scan_result->nresults++;
+
+	return true;
+}
+
+static inline gdouble
+rspamd_check_group_score(struct rspamd_task *task,
+						 const gchar *symbol,
+						 struct rspamd_symbols_group *gr,
+						 gdouble *group_score,
+						 gdouble w)
+{
+	if (gr != NULL && group_score && gr->max_score > 0.0 && w > 0.0) {
+		if (*group_score >= gr->max_score && w > 0) {
+			msg_info_task("maximum group score %.2f for group %s has been reached,"
+						  " ignoring symbol %s with weight %.2f",
+						  gr->max_score,
+						  gr->name, symbol, w);
+			return NAN;
+		}
+		else if (*group_score + w > gr->max_score) {
+			w = gr->max_score - *group_score;
+		}
+	}
+
+	return w;
+}
+
+#ifndef DBL_EPSILON
+#define DBL_EPSILON 2.2204460492503131e-16
+#endif
+
+static struct rspamd_symbol_result *
+insert_metric_result(struct rspamd_task *task,
+					 const gchar *symbol,
+					 double weight,
+					 const gchar *opt,
+					 struct rspamd_scan_result *metric_res,
+					 enum rspamd_symbol_insert_flags flags,
+					 bool *new_sym)
+{
+	struct rspamd_symbol_result *symbol_result = NULL;
+	gdouble final_score, *gr_score = NULL, next_gf = 1.0, diff;
+	struct rspamd_symbol *sdef;
+	struct rspamd_symbols_group *gr = NULL;
+	const ucl_object_t *mobj, *sobj;
+	gint max_shots = G_MAXINT, ret;
+	guint i;
+	khiter_t k;
+	gboolean single = !!(flags & RSPAMD_SYMBOL_INSERT_SINGLE);
+	gchar *sym_cpy;
+
+	if (!isfinite(weight)) {
+		msg_warn_task("detected %s score for symbol %s, replace it with zero",
+					  isnan(weight) ? "NaN" : "infinity", symbol);
+		weight = 0.0;
+	}
+
+	msg_debug_metric("want to insert symbol %s, initial weight %.2f",
+					 symbol, weight);
+
+	sdef = g_hash_table_lookup(task->cfg->symbols, symbol);
+	if (sdef == NULL) {
+		if (flags & RSPAMD_SYMBOL_INSERT_ENFORCE) {
+			final_score = 1.0 * weight; /* Enforce static weight to 1.0 */
+		}
+		else {
+			final_score = 0.0;
+		}
+
+		msg_debug_metric("no symbol definition for %s; final multiplier %.2f",
+						 symbol, final_score);
+	}
+	else {
+		if (sdef->cache_item) {
+			/* Check if we can insert this symbol at all */
+			if (!rspamd_symcache_is_item_allowed(task, sdef->cache_item, FALSE)) {
+				msg_debug_metric("symbol %s is not allowed to be inserted due to settings",
+								 symbol);
+				return NULL;
+			}
+		}
+
+		final_score = (*sdef->weight_ptr) * weight;
+
+		PTR_ARRAY_FOREACH(sdef->groups, i, gr)
+		{
+			k = kh_get(rspamd_symbols_group_hash, metric_res->sym_groups, gr);
+
+			if (k == kh_end(metric_res->sym_groups)) {
+				k = kh_put(rspamd_symbols_group_hash, metric_res->sym_groups,
+						   gr, &ret);
+				kh_value(metric_res->sym_groups, k) = 0;
+			}
+		}
+
+		msg_debug_metric("metric multiplier for %s is %.2f",
+						 symbol, *sdef->weight_ptr);
+	}
+
+	if (task->settings) {
+		gdouble corr;
+		mobj = ucl_object_lookup(task->settings, "scores");
+
+		if (!mobj) {
+			/* Legacy */
+			mobj = task->settings;
+		}
+		else {
+			msg_debug_metric("found scores in the settings");
+		}
+
+		sobj = ucl_object_lookup(mobj, symbol);
+		if (sobj != NULL && ucl_object_todouble_safe(sobj, &corr)) {
+			msg_debug_metric("settings: changed weight of symbol %s from %.2f "
+							 "to %.2f * %.2f",
+							 symbol, final_score, corr, weight);
+			final_score = corr * weight;
+		}
+	}
+
+	k = kh_get(rspamd_symbols_hash, metric_res->symbols, symbol);
+	if (k != kh_end(metric_res->symbols)) {
+		/* Existing metric score */
+		symbol_result = kh_value(metric_res->symbols, k);
+		if (single) {
+			max_shots = 1;
+		}
+		else {
+			if (sdef) {
+				if (sdef->groups) {
+					PTR_ARRAY_FOREACH(sdef->groups, i, gr)
+					{
+						if (gr->flags & RSPAMD_SYMBOL_GROUP_ONE_SHOT) {
+							max_shots = 1;
+						}
+					}
+				}
+
+				max_shots = MIN(max_shots, sdef->nshots);
+			}
+			else {
+				max_shots = task->cfg->default_max_shots;
+			}
+		}
+
+		msg_debug_metric("nshots: %d for symbol %s", max_shots, symbol);
+
+		if (!single && (max_shots > 0 && (symbol_result->nshots >= max_shots))) {
+			single = TRUE;
+		}
+
+		symbol_result->nshots++;
+
+		if (opt) {
+			rspamd_task_add_result_option(task, symbol_result, opt, strlen(opt));
+		}
+
+		/* Adjust diff */
+		if (!single) {
+			diff = final_score;
+			msg_debug_metric("symbol %s can be inserted multiple times: %.2f weight",
+							 symbol, diff);
+		}
+		else {
+			if (fabs(symbol_result->score) < fabs(final_score) &&
+				signbit(symbol_result->score) == signbit(final_score)) {
+				/* Replace less significant weight with a more significant one */
+				diff = final_score - symbol_result->score;
+				msg_debug_metric("symbol %s can be inserted single time;"
+								 " weight adjusted %.2f + %.2f",
+								 symbol, symbol_result->score, diff);
+			}
+			else {
+				diff = 0;
+			}
+		}
+
+		if (diff) {
+			/* Handle grow factor */
+			if (metric_res->grow_factor && diff > 0) {
+				diff *= metric_res->grow_factor;
+				next_gf *= task->cfg->grow_factor;
+			}
+			else if (diff > 0) {
+				next_gf = task->cfg->grow_factor;
+			}
+
+			msg_debug_metric("adjust grow factor to %.2f for symbol %s (%.2f final)",
+							 next_gf, symbol, diff);
+
+			if (sdef) {
+				PTR_ARRAY_FOREACH(sdef->groups, i, gr)
+				{
+					gdouble cur_diff;
+
+					k = kh_get(rspamd_symbols_group_hash,
+							   metric_res->sym_groups, gr);
+					g_assert(k != kh_end(metric_res->sym_groups));
+					gr_score = &kh_value(metric_res->sym_groups, k);
+					cur_diff = rspamd_check_group_score(task, symbol, gr,
+														gr_score, diff);
+
+					if (isnan(cur_diff)) {
+						/* Limit reached, do not add result */
+						msg_debug_metric(
+							"group limit %.2f is reached for %s when inserting symbol %s;"
+							" drop score %.2f",
+							*gr_score, gr->name, symbol, diff);
+
+						diff = NAN;
+						break;
+					}
+					else if (gr_score) {
+						*gr_score += cur_diff;
+
+						if (cur_diff < diff) {
+							/* Reduce */
+							msg_debug_metric(
+								"group limit %.2f is reached for %s when inserting symbol %s;"
+								" reduce score %.2f - %.2f",
+								*gr_score, gr->name, symbol, diff, cur_diff);
+							diff = cur_diff;
+						}
+					}
+				}
+			}
+
+			if (!isnan(diff)) {
+				metric_res->score += diff;
+				metric_res->grow_factor = next_gf;
+
+				if (single) {
+					msg_debug_metric("final score for single symbol %s = %.2f; %.2f diff",
+									 symbol, final_score, diff);
+					symbol_result->score = final_score;
+				}
+				else {
+					msg_debug_metric("increase final score for multiple symbol %s += %.2f = %.2f",
+									 symbol, symbol_result->score, diff);
+					symbol_result->score += diff;
+				}
+			}
+		}
+	}
+	else {
+		/* New result */
+		if (new_sym) {
+			*new_sym = true;
+		}
+
+		sym_cpy = rspamd_mempool_strdup(task->task_pool, symbol);
+		k = kh_put(rspamd_symbols_hash, metric_res->symbols,
+				   sym_cpy, &ret);
+		g_assert(ret > 0);
+		symbol_result = rspamd_mempool_alloc0(task->task_pool, sizeof(*symbol_result));
+		kh_value(metric_res->symbols, k) = symbol_result;
+
+		/* Handle grow factor */
+		if (metric_res->grow_factor && final_score > 0) {
+			final_score *= metric_res->grow_factor;
+			next_gf *= task->cfg->grow_factor;
+		}
+		else if (final_score > 0) {
+			next_gf = task->cfg->grow_factor;
+		}
+
+		msg_debug_metric("adjust grow factor to %.2f for symbol %s (%.2f final)",
+						 next_gf, symbol, final_score);
+
+		symbol_result->name = sym_cpy;
+		symbol_result->sym = sdef;
+		symbol_result->nshots = 1;
+
+		if (sdef) {
+			/* Check group limits */
+			PTR_ARRAY_FOREACH(sdef->groups, i, gr)
+			{
+				gdouble cur_score;
+
+				k = kh_get(rspamd_symbols_group_hash, metric_res->sym_groups, gr);
+				g_assert(k != kh_end(metric_res->sym_groups));
+				gr_score = &kh_value(metric_res->sym_groups, k);
+				cur_score = rspamd_check_group_score(task, symbol, gr,
+													 gr_score, final_score);
+
+				if (isnan(cur_score)) {
+					/* Limit reached, do not add result */
+					msg_debug_metric(
+						"group limit %.2f is reached for %s when inserting symbol %s;"
+						" drop score %.2f",
+						*gr_score, gr->name, symbol, final_score);
+					final_score = NAN;
+					break;
+				}
+				else if (gr_score) {
+					*gr_score += cur_score;
+
+					if (cur_score < final_score) {
+						/* Reduce */
+						msg_debug_metric(
+							"group limit %.2f is reached for %s when inserting symbol %s;"
+							" reduce score %.2f - %.2f",
+							*gr_score, gr->name, symbol, final_score, cur_score);
+						final_score = cur_score;
+					}
+				}
+			}
+		}
+
+		if (!isnan(final_score)) {
+			const double epsilon = DBL_EPSILON;
+
+			metric_res->score += final_score;
+			metric_res->grow_factor = next_gf;
+			symbol_result->score = final_score;
+
+			if (final_score > epsilon) {
+				metric_res->npositive++;
+				metric_res->positive_score += final_score;
+			}
+			else if (final_score < -epsilon) {
+				metric_res->nnegative++;
+				metric_res->negative_score += fabs(final_score);
+			}
+		}
+		else {
+			symbol_result->score = 0;
+		}
+
+		if (opt) {
+			rspamd_task_add_result_option(task, symbol_result, opt, strlen(opt));
+		}
+	}
+
+	msg_debug_metric("final insertion for symbol %s, score %.2f, factor: %f",
+					 symbol,
+					 symbol_result->score,
+					 final_score);
+	metric_res->nresults++;
+
+	return symbol_result;
+}
+
+struct rspamd_symbol_result *
+rspamd_task_insert_result_full(struct rspamd_task *task,
+							   const gchar *symbol,
+							   double weight,
+							   const gchar *opt,
+							   enum rspamd_symbol_insert_flags flags,
+							   struct rspamd_scan_result *result)
+{
+	struct rspamd_symbol_result *symbol_result = NULL, *ret = NULL;
+	struct rspamd_scan_result *mres;
+
+	/*
+	 * We allow symbols to be inserted for skipped tasks, as it might be a
+	 * race condition before some symbol is finished and skip flag being set.
+	 */
+	if (!RSPAMD_TASK_IS_SKIPPED(task) && (task->processed_stages & (RSPAMD_TASK_STAGE_IDEMPOTENT >> 1))) {
+		msg_err_task("cannot insert symbol %s on idempotent phase",
+					 symbol);
+
+		return NULL;
+	}
+
+	if (result == NULL) {
+		/* Insert everywhere */
+		DL_FOREACH(task->result, mres)
+		{
+			if (mres->symbol_cbref != -1) {
+				/* Check if we can insert this symbol to this symbol result */
+				GError *err = NULL;
+				lua_State *L = (lua_State *) task->cfg->lua_state;
+
+				if (!rspamd_lua_universal_pcall(L, mres->symbol_cbref,
+												G_STRLOC, 1, "uss", &err,
+												"rspamd{task}", task, symbol, mres->name ? mres->name : "default")) {
+					msg_warn_task("cannot call for symbol_cbref for result %s: %e",
+								  mres->name ? mres->name : "default", err);
+					g_error_free(err);
+
+					continue;
+				}
+				else {
+					if (!lua_toboolean(L, -1)) {
+						/* Skip symbol */
+						msg_debug_metric("skip symbol %s for result %s due to Lua return value",
+										 symbol, mres->name);
+						lua_pop(L, 1); /* Remove result */
+
+						continue;
+					}
+
+					lua_pop(L, 1); /* Remove result */
+				}
+			}
+
+			bool new_symbol = false;
+
+			symbol_result = insert_metric_result(task,
+												 symbol,
+												 weight,
+												 opt,
+												 mres,
+												 flags,
+												 &new_symbol);
+
+			if (mres->name == NULL) {
+				/* Default result */
+				ret = symbol_result;
+
+				/* Process cache item */
+				if (symbol_result && task->cfg->cache && symbol_result->sym && symbol_result->nshots == 1) {
+					rspamd_symcache_inc_frequency(task->cfg->cache,
+												  symbol_result->sym->cache_item,
+												  symbol_result->sym->name);
+				}
+			}
+			else if (new_symbol) {
+				/* O(N) but we normally don't have any shadow results */
+				LL_APPEND(ret, symbol_result);
+			}
+		}
+	}
+	else {
+		/* Specific insertion */
+		symbol_result = insert_metric_result(task,
+											 symbol,
+											 weight,
+											 opt,
+											 result,
+											 flags,
+											 NULL);
+		ret = symbol_result;
+
+		if (result->name == NULL) {
+			/* Process cache item */
+			if (symbol_result && task->cfg->cache && symbol_result->sym && symbol_result->nshots == 1) {
+				rspamd_symcache_inc_frequency(task->cfg->cache,
+											  symbol_result->sym->cache_item,
+											  symbol_result->sym->name);
+			}
+		}
+	}
+
+	return ret;
+}
+
+static gchar *
+rspamd_task_option_safe_copy(struct rspamd_task *task,
+							 const gchar *val,
+							 gsize vlen,
+							 gsize *outlen)
+{
+	const gchar *p, *end;
+
+	p = val;
+	end = val + vlen;
+	vlen = 0; /* Reuse */
+
+	while (p < end) {
+		if (*p & 0x80) {
+			UChar32 uc;
+			gint off = 0;
+
+			U8_NEXT(p, off, end - p, uc);
+
+			if (uc > 0) {
+				if (u_isprint(uc)) {
+					vlen += off;
+				}
+				else {
+					/* We will replace it with 0xFFFD */
+					vlen += MAX(off, 3);
+				}
+			}
+			else {
+				vlen += MAX(off, 3);
+			}
+
+			p += off;
+		}
+		else if (!g_ascii_isprint(*p)) {
+			/* Another 0xFFFD */
+			vlen += 3;
+			p++;
+		}
+		else {
+			p++;
+			vlen++;
+		}
+	}
+
+	gchar *dest, *d;
+
+	dest = rspamd_mempool_alloc(task->task_pool, vlen + 1);
+	d = dest;
+	p = val;
+
+	while (p < end) {
+		if (*p & 0x80) {
+			UChar32 uc;
+			gint off = 0;
+
+			U8_NEXT(p, off, end - p, uc);
+
+			if (uc > 0) {
+				if (u_isprint(uc)) {
+					memcpy(d, p, off);
+					d += off;
+				}
+				else {
+					/* We will replace it with 0xFFFD */
+					*d++ = '\357';
+					*d++ = '\277';
+					*d++ = '\275';
+				}
+			}
+			else {
+				*d++ = '\357';
+				*d++ = '\277';
+				*d++ = '\275';
+			}
+
+			p += off;
+		}
+		else if (!g_ascii_isprint(*p)) {
+			/* Another 0xFFFD */
+			*d++ = '\357';
+			*d++ = '\277';
+			*d++ = '\275';
+			p++;
+		}
+		else {
+			*d++ = *p++;
+		}
+	}
+
+	*d = '\0';
+	*(outlen) = d - dest;
+
+	return dest;
+}
+
+gboolean
+rspamd_task_add_result_option(struct rspamd_task *task,
+							  struct rspamd_symbol_result *s,
+							  const gchar *val,
+							  gsize vlen)
+{
+	struct rspamd_symbol_option *opt, srch;
+	gboolean ret = FALSE;
+	gchar *opt_cpy = NULL;
+	gsize cpy_len;
+	khiter_t k;
+	gint r;
+	struct rspamd_symbol_result *cur;
+
+	if (s && val) {
+		/*
+		 * Here we assume that this function is all the time called with the
+		 * symbol from the default result, not some shadow result, or
+		 * the option insertion will be wrong
+		 */
+		LL_FOREACH(s, cur)
+		{
+			if (cur->opts_len < 0) {
+				/* Cannot add more options, give up */
+				msg_debug_task("cannot add more options to symbol %s when adding option %s",
+							   cur->name, val);
+				ret = FALSE;
+				continue;
+			}
+
+			if (!cur->options) {
+				cur->options = kh_init(rspamd_options_hash);
+			}
+
+			if (vlen + cur->opts_len > task->cfg->max_opts_len) {
+				/* Add truncated option */
+				msg_info_task("cannot add more options to symbol %s when adding option %s",
+							  cur->name, val);
+				val = "...";
+				vlen = 3;
+				cur->opts_len = -1;
+			}
+
+			if (!(cur->sym && (cur->sym->flags & RSPAMD_SYMBOL_FLAG_ONEPARAM))) {
+
+				srch.option = (gchar *) val;
+				srch.optlen = vlen;
+				k = kh_get(rspamd_options_hash, cur->options, &srch);
+
+				if (k == kh_end(cur->options)) {
+					opt_cpy = rspamd_task_option_safe_copy(task, val, vlen, &cpy_len);
+					if (cpy_len != vlen) {
+						srch.option = (gchar *) opt_cpy;
+						srch.optlen = cpy_len;
+						k = kh_get(rspamd_options_hash, cur->options, &srch);
+					}
+					/* Append new options */
+					if (k == kh_end(cur->options)) {
+						opt = rspamd_mempool_alloc0(task->task_pool, sizeof(*opt));
+						opt->optlen = cpy_len;
+						opt->option = opt_cpy;
+
+						kh_put(rspamd_options_hash, cur->options, opt, &r);
+						DL_APPEND(cur->opts_head, opt);
+
+						if (s == cur) {
+							ret = TRUE;
+						}
+					}
+				}
+			}
+			else {
+				/* Skip addition */
+				if (s == cur) {
+					ret = FALSE;
+				}
+			}
+
+			if (ret && cur->opts_len >= 0) {
+				cur->opts_len += vlen;
+			}
+		}
+	}
+	else if (!val) {
+		ret = TRUE;
+	}
+
+	task->result->nresults++;
+
+	return ret;
+}
+
+struct rspamd_action_config *
+rspamd_find_action_config_for_action(struct rspamd_scan_result *scan_result,
+									 struct rspamd_action *act)
+{
+	for (unsigned int i = 0; i < scan_result->nactions; i++) {
+		struct rspamd_action_config *cur = &scan_result->actions_config[i];
+
+		if (act == cur->action) {
+			return cur;
+		}
+	}
+
+	return NULL;
+}
+
+struct rspamd_action *
+rspamd_check_action_metric(struct rspamd_task *task,
+						   struct rspamd_passthrough_result **ppr,
+						   struct rspamd_scan_result *scan_result)
+{
+	struct rspamd_action_config *action_lim,
+		*noaction = NULL;
+	struct rspamd_action *selected_action = NULL, *least_action = NULL;
+	struct rspamd_passthrough_result *pr, *sel_pr = NULL;
+	double max_score = -(G_MAXDOUBLE), sc;
+	gboolean seen_least = FALSE;
+
+	if (scan_result == NULL) {
+		scan_result = task->result;
+	}
+
+	if (scan_result->passthrough_result != NULL) {
+		DL_FOREACH(scan_result->passthrough_result, pr)
+		{
+			struct rspamd_action_config *act_config =
+				rspamd_find_action_config_for_action(scan_result, pr->action);
+
+			/* Skip disabled actions */
+			if (act_config && (act_config->flags & RSPAMD_ACTION_RESULT_DISABLED)) {
+				continue;
+			}
+
+			if (!seen_least || !(pr->flags & RSPAMD_PASSTHROUGH_LEAST)) {
+				sc = pr->target_score;
+				selected_action = pr->action;
+
+				if (!(pr->flags & RSPAMD_PASSTHROUGH_LEAST)) {
+					if (!isnan(sc)) {
+						if (pr->action->action_type == METRIC_ACTION_NOACTION) {
+							scan_result->score = MIN(sc, scan_result->score);
+						}
+						else {
+							scan_result->score = sc;
+						}
+					}
+
+					if (ppr) {
+						*ppr = pr;
+					}
+
+					return selected_action;
+				}
+				else {
+					seen_least = true;
+					least_action = selected_action;
+
+					if (isnan(sc)) {
+
+						if (selected_action->flags & RSPAMD_ACTION_NO_THRESHOLD) {
+							/*
+							 * In this case, we have a passthrough action that
+							 * is `least` action, however, there is no threshold
+							 * on it.
+							 *
+							 * Hence, we imply the following logic:
+							 *
+							 * - we leave score unchanged
+							 * - we apply passthrough no threshold action unless
+							 *   score based action *is not* reject, otherwise
+							 *   we apply reject action
+							 */
+						}
+						else {
+							sc = selected_action->threshold;
+							max_score = sc;
+							sel_pr = pr;
+						}
+					}
+					else {
+						max_score = sc;
+						sel_pr = pr;
+					}
+				}
+			}
+		}
+	}
+
+	/*
+	 * Select result by score
+	 */
+	for (size_t i = scan_result->nactions - 1; i != (size_t) -1; i--) {
+		action_lim = &scan_result->actions_config[i];
+		sc = action_lim->cur_limit;
+
+		if (action_lim->action->action_type == METRIC_ACTION_NOACTION) {
+			noaction = action_lim;
+		}
+
+		if ((action_lim->flags & (RSPAMD_ACTION_RESULT_DISABLED | RSPAMD_ACTION_RESULT_NO_THRESHOLD))) {
+			continue;
+		}
+
+		if (isnan(sc) ||
+			(action_lim->action->flags & (RSPAMD_ACTION_NO_THRESHOLD | RSPAMD_ACTION_HAM))) {
+			continue;
+		}
+
+		if (scan_result->score >= sc && sc > max_score) {
+			selected_action = action_lim->action;
+			max_score = sc;
+		}
+	}
+
+	if (selected_action == NULL) {
+		selected_action = noaction->action;
+	}
+
+	if (selected_action) {
+
+		if (seen_least) {
+			/* Adjust least action */
+			if (least_action->flags & RSPAMD_ACTION_NO_THRESHOLD) {
+				if (selected_action->action_type != METRIC_ACTION_REJECT &&
+					selected_action->action_type != METRIC_ACTION_DISCARD) {
+					/* Override score based action with least action */
+					selected_action = least_action;
+
+					if (ppr) {
+						*ppr = sel_pr;
+					}
+				}
+			}
+			else {
+				/* Adjust score if needed */
+				if (max_score > scan_result->score) {
+					if (ppr) {
+						*ppr = sel_pr;
+					}
+
+					scan_result->score = max_score;
+				}
+			}
+		}
+
+		return selected_action;
+	}
+
+	if (ppr) {
+		*ppr = sel_pr;
+	}
+
+	return noaction->action;
+}
+
+struct rspamd_symbol_result *
+rspamd_task_find_symbol_result(struct rspamd_task *task, const char *sym,
+							   struct rspamd_scan_result *result)
+{
+	struct rspamd_symbol_result *res = NULL;
+	khiter_t k;
+
+	if (result == NULL) {
+		/* Use default result */
+		result = task->result;
+	}
+
+	k = kh_get(rspamd_symbols_hash, result->symbols, sym);
+
+	if (k != kh_end(result->symbols)) {
+		res = kh_value(result->symbols, k);
+	}
+
+	return res;
+}
+
+struct rspamd_symbol_result *rspamd_task_remove_symbol_result(
+	struct rspamd_task *task,
+	const gchar *symbol,
+	struct rspamd_scan_result *result)
+{
+	struct rspamd_symbol_result *res = NULL;
+	khiter_t k;
+
+	if (result == NULL) {
+		/* Use default result */
+		result = task->result;
+	}
+
+	k = kh_get(rspamd_symbols_hash, result->symbols, symbol);
+
+	if (k != kh_end(result->symbols)) {
+		res = kh_value(result->symbols, k);
+
+		if (!isnan(res->score)) {
+			/* Remove score from the result */
+			result->score -= res->score;
+
+			/* Also check the group limit */
+			if (result->sym_groups && res->sym) {
+				struct rspamd_symbol_group *gr;
+				gint i;
+				khiter_t k_groups;
+
+				PTR_ARRAY_FOREACH(res->sym->groups, i, gr)
+				{
+					gdouble *gr_score;
+
+					k_groups = kh_get(rspamd_symbols_group_hash,
+									  result->sym_groups, gr);
+
+					if (k_groups != kh_end(result->sym_groups)) {
+						gr_score = &kh_value(result->sym_groups, k_groups);
+
+						if (gr_score) {
+							*gr_score -= res->score;
+						}
+					}
+				}
+			}
+		}
+
+		kh_del(rspamd_symbols_hash, result->symbols, k);
+	}
+	else {
+		return NULL;
+	}
+
+	return res;
+}
+
+void rspamd_task_symbol_result_foreach(struct rspamd_task *task,
+									   struct rspamd_scan_result *result, GHFunc func,
+									   gpointer ud)
+{
+	const gchar *kk;
+	struct rspamd_symbol_result *res;
+
+	if (result == NULL) {
+		/* Use default result */
+		result = task->result;
+	}
+
+	if (func) {
+		kh_foreach(result->symbols, kk, res, {
+			func((gpointer) kk, (gpointer) res, ud);
+		});
+	}
+}
+
+struct rspamd_scan_result *
+rspamd_find_metric_result(struct rspamd_task *task,
+						  const gchar *name)
+{
+	struct rspamd_scan_result *res;
+
+	if (name == NULL || strcmp(name, "default") == 0) {
+		return task->result;
+	}
+
+	DL_FOREACH(task->result, res)
+	{
+		if (res->name && strcmp(res->name, name) == 0) {
+			return res;
+		}
+	}
+
+	return NULL;
+}
diff --git a/src/libmime/scan_result.h b/src/libmime/scan_result.h
new file mode 100644
index 0000000..46c2de8
--- /dev/null
+++ b/src/libmime/scan_result.h
@@ -0,0 +1,250 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * @file scan_result.h
+ * Scan result holder
+ */
+
+#ifndef RSPAMD_SCAN_RESULT_H
+#define RSPAMD_SCAN_RESULT_H
+
+#include "config.h"
+#include "rspamd_symcache.h"
+#include "task.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_settings;
+struct rspamd_classifier_config;
+
+struct rspamd_symbol_option {
+	gchar *option;
+	gsize optlen;
+	struct rspamd_symbol_option *prev, *next;
+};
+
+enum rspamd_symbol_result_flags {
+	RSPAMD_SYMBOL_RESULT_NORMAL = 0,
+	RSPAMD_SYMBOL_RESULT_IGNORED = (1 << 0)
+};
+
+struct kh_rspamd_options_hash_s;
+
+/**
+ * Rspamd symbol
+ */
+struct rspamd_symbol_result {
+	double score;                             /**< symbol's score							*/
+	struct kh_rspamd_options_hash_s *options; /**< list of symbol's options				*/
+	struct rspamd_symbol_option *opts_head;   /**< head of linked list of options			*/
+	const gchar *name;
+	struct rspamd_symbol *sym; /**< symbol configuration					*/
+	gssize opts_len;           /**< total size of all options (negative if truncated option is added) */
+	guint nshots;
+	int flags;
+	struct rspamd_symbol_result *next; /**< for shadow results */
+};
+
+
+#define RSPAMD_PASSTHROUGH_NORMAL 1
+#define RSPAMD_PASSTHROUGH_LOW 0
+#define RSPAMD_PASSTHROUGH_HIGH 2
+#define RSPAMD_PASSTHROUGH_CRITICAL 3
+
+#define RSPAMD_PASSTHROUGH_LEAST (1u << 0u)
+#define RSPAMD_PASSTHROUGH_NO_SMTP_MESSAGE (1u << 1u)
+#define RSPAMD_PASSTHROUGH_PROCESS_ALL (1u << 2u)
+
+struct rspamd_passthrough_result {
+	struct rspamd_action *action;
+	guint priority;
+	guint flags;
+	double target_score;
+	const gchar *message;
+	const gchar *module;
+	struct rspamd_passthrough_result *prev, *next;
+};
+
+
+enum rspamd_action_config_flags {
+	RSPAMD_ACTION_RESULT_DEFAULT = 0,
+	RSPAMD_ACTION_RESULT_NO_THRESHOLD = (1u << 0u),
+	RSPAMD_ACTION_RESULT_DISABLED = (1u << 1u),
+};
+struct rspamd_action_config {
+	gdouble cur_limit;
+	int flags;
+	struct rspamd_action *action;
+};
+
+struct kh_rspamd_symbols_hash_s;
+struct kh_rspamd_symbols_group_hash_s;
+
+
+struct rspamd_scan_result {
+	double score;       /**< total score							*/
+	double grow_factor; /**< current grow factor					*/
+	struct rspamd_passthrough_result *passthrough_result;
+	double positive_score;
+	double negative_score;
+	struct kh_rspamd_symbols_hash_s *symbols;          /**< symbols of metric						*/
+	struct kh_rspamd_symbols_group_hash_s *sym_groups; /**< groups of symbols						*/
+	struct rspamd_action_config *actions_config;
+	const gchar *name;        /**< for named results, NULL is the default result */
+	struct rspamd_task *task; /**< back reference */
+	gint symbol_cbref;        /**< lua function that defines if a symbol can be inserted, -1 if unused */
+	guint nactions;
+	guint npositive;
+	guint nnegative;
+	guint nresults;                         /**< all results: positive, negative, passthrough etc */
+	guint nresults_postfilters;             /**< how many results are there before postfilters stage */
+	struct rspamd_scan_result *prev, *next; /**< double linked list of results */
+};
+
+/**
+ * Create or return existing result for the specified metric name
+ * @param task task object
+ * @return metric result or NULL if metric `name` has not been found
+ */
+struct rspamd_scan_result *rspamd_create_metric_result(struct rspamd_task *task,
+													   const gchar *name, gint lua_sym_cbref);
+
+/**
+ * Find result with a specific name (NULL means the default result)
+ * @param task
+ * @param name
+ * @return
+ */
+struct rspamd_scan_result *rspamd_find_metric_result(struct rspamd_task *task,
+													 const gchar *name);
+
+/**
+ * Adds a new passthrough result to a task
+ * @param task
+ * @param action
+ * @param priority
+ * @param target_score
+ * @param message
+ * @param module
+ */
+bool rspamd_add_passthrough_result(struct rspamd_task *task,
+								   struct rspamd_action *action, guint priority,
+								   double target_score, const gchar *message,
+								   const gchar *module, guint flags,
+								   struct rspamd_scan_result *scan_result);
+
+enum rspamd_symbol_insert_flags {
+	RSPAMD_SYMBOL_INSERT_DEFAULT = 0,
+	RSPAMD_SYMBOL_INSERT_SINGLE = (1 << 0),
+	RSPAMD_SYMBOL_INSERT_ENFORCE = (1 << 1),
+};
+
+/**
+ * Insert a result to task
+ * @param task worker's task that present message from user
+ * @param metric_name metric's name to which we need to insert result
+ * @param symbol symbol to insert
+ * @param weight numeric weight for symbol
+ * @param opts list of symbol's options
+ */
+struct rspamd_symbol_result *rspamd_task_insert_result_full(struct rspamd_task *task,
+															const gchar *symbol,
+															double weight,
+															const gchar *opts,
+															enum rspamd_symbol_insert_flags flags,
+															struct rspamd_scan_result *result);
+
+#define rspamd_task_insert_result_single(task, symbol, weight, opts) \
+	rspamd_task_insert_result_full((task), (symbol), (weight), (opts), RSPAMD_SYMBOL_INSERT_SINGLE, NULL)
+#define rspamd_task_insert_result(task, symbol, weight, opts) \
+	rspamd_task_insert_result_full((task), (symbol), (weight), (opts), RSPAMD_SYMBOL_INSERT_DEFAULT, NULL)
+
+/**
+ * Removes a symbol from a specific symbol result
+ * @param task
+ * @param symbol
+ * @param result
+ * @return
+ */
+struct rspamd_symbol_result *rspamd_task_remove_symbol_result(
+	struct rspamd_task *task,
+	const gchar *symbol,
+	struct rspamd_scan_result *result);
+/**
+ * Adds new option to symbol
+ * @param task
+ * @param s
+ * @param opt
+ */
+gboolean rspamd_task_add_result_option(struct rspamd_task *task,
+									   struct rspamd_symbol_result *s,
+									   const gchar *opt,
+									   gsize vlen);
+
+/**
+ * Finds symbol result
+ * @param task
+ * @param sym
+ * @return
+ */
+struct rspamd_symbol_result *
+rspamd_task_find_symbol_result(struct rspamd_task *task, const char *sym,
+							   struct rspamd_scan_result *result);
+
+/**
+ * Compatibility function to iterate on symbols hash
+ * @param task
+ * @param func
+ * @param ud
+ */
+void rspamd_task_symbol_result_foreach(struct rspamd_task *task,
+									   struct rspamd_scan_result *result,
+									   GHFunc func,
+									   gpointer ud);
+
+/**
+ * Default consolidation function for metric, it get all symbols and multiply symbol
+ * weight by some factor that is specified in config. Default factor is 1.
+ * @param task worker's task that present message from user
+ * @param metric_name name of metric
+ * @return result metric weight
+ */
+double rspamd_factor_consolidation_func(struct rspamd_task *task,
+										const gchar *metric_name,
+										const gchar *unused);
+
+
+/**
+ * Check thresholds and return action for a task
+ * @param task
+ * @return
+ */
+struct rspamd_action *rspamd_check_action_metric(struct rspamd_task *task,
+												 struct rspamd_passthrough_result **ppr,
+												 struct rspamd_scan_result *scan_result);
+
+struct rspamd_action_config *rspamd_find_action_config_for_action(struct rspamd_scan_result *scan_result,
+																  struct rspamd_action *act);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif
diff --git a/src/libmime/scan_result_private.h b/src/libmime/scan_result_private.h
new file mode 100644
index 0000000..cf0c0c5
--- /dev/null
+++ b/src/libmime/scan_result_private.h
@@ -0,0 +1,55 @@
+//
+// Created by Vsevolod Stakhov on 2019-01-14.
+//
+
+#ifndef RSPAMD_SCAN_RESULT_PRIVATE_H
+#define RSPAMD_SCAN_RESULT_PRIVATE_H
+
+#include "scan_result.h"
+#include "contrib/libucl/khash.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RSPAMD_OPTS_SEED 0x9f1f608628a4fefbULL
+#define rspamd_symopt_hash(opt) (rspamd_cryptobox_fast_hash( \
+	((struct rspamd_symbol_option *) opt)->option,           \
+	((struct rspamd_symbol_option *) opt)->optlen, RSPAMD_OPTS_SEED))
+static inline bool
+rspamd_symopt_equal(const struct rspamd_symbol_option *o1,
+					const struct rspamd_symbol_option *o2)
+{
+	if (o1->optlen == o2->optlen) {
+		return (memcmp(o1->option, o2->option, o1->optlen) == 0);
+	}
+
+	return false;
+}
+
+KHASH_INIT(rspamd_options_hash, struct rspamd_symbol_option *, char,
+		   0, rspamd_symopt_hash, rspamd_symopt_equal);
+/**
+ * Result of metric processing
+ */
+KHASH_MAP_INIT_STR(rspamd_symbols_hash, struct rspamd_symbol_result *);
+#if UINTPTR_MAX <= UINT_MAX
+/* 32 bit */
+#define rspamd_ptr_hash_func(key) (khint32_t)(((uintptr_t) (key)) >> 1)
+#else
+/* likely 64 bit */
+#define rspamd_ptr_hash_func(key) (khint32_t)(((uintptr_t) (key)) >> 3)
+#endif
+#define rspamd_ptr_equal_func(a, b) ((a) == (b))
+KHASH_INIT(rspamd_symbols_group_hash,
+		   void *,
+		   double,
+		   1,
+		   rspamd_ptr_hash_func,
+		   rspamd_ptr_equal_func);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif//RSPAMD_SCAN_RESULT_PRIVATE_H
diff --git a/src/libmime/smtp_parsers.h b/src/libmime/smtp_parsers.h
new file mode 100644
index 0000000..e188b63
--- /dev/null
+++ b/src/libmime/smtp_parsers.h
@@ -0,0 +1,51 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#ifndef SRC_LIBMIME_SMTP_PARSERS_H_
+#define SRC_LIBMIME_SMTP_PARSERS_H_
+
+#include "config.h"
+#include "email_addr.h"
+#include "content_type.h"
+#include "task.h"
+#include "message.h"
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int rspamd_smtp_addr_parse(const char *data, size_t len,
+						   struct rspamd_email_address *addr);
+
+gboolean rspamd_content_disposition_parser(const char *data, size_t len,
+										   struct rspamd_content_disposition *cd,
+										   rspamd_mempool_t *pool);
+
+gboolean
+rspamd_rfc2047_parser(const gchar *in, gsize len, gint *pencoding,
+					  const gchar **charset, gsize *charset_len,
+					  const gchar **encoded, gsize *encoded_len);
+
+rspamd_inet_addr_t *rspamd_parse_smtp_ip(const char *data, size_t len,
+										 rspamd_mempool_t *pool);
+
+guint64 rspamd_parse_smtp_date(const unsigned char *data, size_t len, GError **err);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* SRC_LIBMIME_SMTP_PARSERS_H_ */