Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 1250 insertions, 0 deletions

diff --git a/src/libstat/stat_process.c b/src/libstat/stat_process.c
new file mode 100644
index 0000000..8c1d8ff
--- /dev/null
+++ b/src/libstat/stat_process.c
@@ -0,0 +1,1250 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+#include "config.h"
+#include "stat_api.h"
+#include "rspamd.h"
+#include "stat_internal.h"
+#include "libmime/message.h"
+#include "libmime/images.h"
+#include "libserver/html/html.h"
+#include "lua/lua_common.h"
+#include "libserver/mempool_vars_internal.h"
+#include "utlist.h"
+#include <math.h>
+
+#define RSPAMD_CLASSIFY_OP 0
+#define RSPAMD_LEARN_OP 1
+#define RSPAMD_UNLEARN_OP 2
+
+static const gdouble similarity_threshold = 80.0;
+
+static void
+rspamd_stat_tokenize_parts_metadata(struct rspamd_stat_ctx *st_ctx,
+									struct rspamd_task *task)
+{
+	GArray *ar;
+	rspamd_stat_token_t elt;
+	guint i;
+	lua_State *L = task->cfg->lua_state;
+
+	ar = g_array_sized_new(FALSE, FALSE, sizeof(elt), 16);
+	memset(&elt, 0, sizeof(elt));
+	elt.flags = RSPAMD_STAT_TOKEN_FLAG_META;
+
+	if (st_ctx->lua_stat_tokens_ref != -1) {
+		gint err_idx, ret;
+		struct rspamd_task **ptask;
+
+		lua_pushcfunction(L, &rspamd_lua_traceback);
+		err_idx = lua_gettop(L);
+		lua_rawgeti(L, LUA_REGISTRYINDEX, st_ctx->lua_stat_tokens_ref);
+
+		ptask = lua_newuserdata(L, sizeof(*ptask));
+		*ptask = task;
+		rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+		if ((ret = lua_pcall(L, 1, 1, err_idx)) != 0) {
+			msg_err_task("call to stat_tokens lua "
+						 "script failed (%d): %s",
+						 ret, lua_tostring(L, -1));
+		}
+		else {
+			if (lua_type(L, -1) != LUA_TTABLE) {
+				msg_err_task("stat_tokens invocation must return "
+							 "table and not %s",
+							 lua_typename(L, lua_type(L, -1)));
+			}
+			else {
+				guint vlen;
+				rspamd_ftok_t tok;
+
+				vlen = rspamd_lua_table_size(L, -1);
+
+				for (i = 0; i < vlen; i++) {
+					lua_rawgeti(L, -1, i + 1);
+					tok.begin = lua_tolstring(L, -1, &tok.len);
+
+					if (tok.begin && tok.len > 0) {
+						elt.original.begin =
+							rspamd_mempool_ftokdup(task->task_pool, &tok);
+						elt.original.len = tok.len;
+						elt.stemmed.begin = elt.original.begin;
+						elt.stemmed.len = elt.original.len;
+						elt.normalized.begin = elt.original.begin;
+						elt.normalized.len = elt.original.len;
+
+						g_array_append_val(ar, elt);
+					}
+
+					lua_pop(L, 1);
+				}
+			}
+		}
+
+		lua_settop(L, 0);
+	}
+
+
+	if (ar->len > 0) {
+		st_ctx->tokenizer->tokenize_func(st_ctx,
+										 task,
+										 ar,
+										 TRUE,
+										 "M",
+										 task->tokens);
+	}
+
+	rspamd_mempool_add_destructor(task->task_pool,
+								  rspamd_array_free_hard, ar);
+}
+
+/*
+ * Tokenize task using the tokenizer specified
+ */
+void rspamd_stat_process_tokenize(struct rspamd_stat_ctx *st_ctx,
+								  struct rspamd_task *task)
+{
+	struct rspamd_mime_text_part *part;
+	rspamd_cryptobox_hash_state_t hst;
+	rspamd_token_t *st_tok;
+	guint i, reserved_len = 0;
+	gdouble *pdiff;
+	guchar hout[rspamd_cryptobox_HASHBYTES];
+	gchar *b32_hout;
+
+	if (st_ctx == NULL) {
+		st_ctx = rspamd_stat_get_ctx();
+	}
+
+	g_assert(st_ctx != NULL);
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+	{
+		if (!IS_TEXT_PART_EMPTY(part) && part->utf_words != NULL) {
+			reserved_len += part->utf_words->len;
+		}
+		/* XXX: normal window size */
+		reserved_len += 5;
+	}
+
+	task->tokens = g_ptr_array_sized_new(reserved_len);
+	rspamd_mempool_add_destructor(task->task_pool,
+								  rspamd_ptr_array_free_hard, task->tokens);
+	rspamd_mempool_notify_alloc(task->task_pool, reserved_len * sizeof(gpointer));
+	pdiff = rspamd_mempool_get_variable(task->task_pool, "parts_distance");
+
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+	{
+		if (!IS_TEXT_PART_EMPTY(part) && part->utf_words != NULL) {
+			st_ctx->tokenizer->tokenize_func(st_ctx, task,
+											 part->utf_words, IS_TEXT_PART_UTF(part),
+											 NULL, task->tokens);
+		}
+
+
+		if (pdiff != NULL && (1.0 - *pdiff) * 100.0 > similarity_threshold) {
+			msg_debug_bayes("message has two common parts (%.2f), so skip the last one",
+							*pdiff);
+			break;
+		}
+	}
+
+	if (task->meta_words != NULL) {
+		st_ctx->tokenizer->tokenize_func(st_ctx,
+										 task,
+										 task->meta_words,
+										 TRUE,
+										 "SUBJECT",
+										 task->tokens);
+	}
+
+	rspamd_stat_tokenize_parts_metadata(st_ctx, task);
+
+	/* Produce signature */
+	rspamd_cryptobox_hash_init(&hst, NULL, 0);
+
+	PTR_ARRAY_FOREACH(task->tokens, i, st_tok)
+	{
+		rspamd_cryptobox_hash_update(&hst, (guchar *) &st_tok->data,
+									 sizeof(st_tok->data));
+	}
+
+	rspamd_cryptobox_hash_final(&hst, hout);
+	b32_hout = rspamd_encode_base32(hout, sizeof(hout), RSPAMD_BASE32_DEFAULT);
+	/*
+	 * We need to strip it to 32 characters providing ~160 bits of
+	 * hash distribution
+	 */
+	b32_hout[32] = '\0';
+	rspamd_mempool_set_variable(task->task_pool, RSPAMD_MEMPOOL_STAT_SIGNATURE,
+								b32_hout, g_free);
+}
+
+static gboolean
+rspamd_stat_classifier_is_skipped(struct rspamd_task *task,
+								  struct rspamd_classifier *cl, gboolean is_learn, gboolean is_spam)
+{
+	GList *cur = is_learn ? cl->cfg->learn_conditions : cl->cfg->classify_conditions;
+	lua_State *L = task->cfg->lua_state;
+	gboolean ret = FALSE;
+
+	while (cur) {
+		gint cb_ref = GPOINTER_TO_INT(cur->data);
+		gint old_top = lua_gettop(L);
+		gint nargs;
+
+		lua_rawgeti(L, LUA_REGISTRYINDEX, cb_ref);
+		/* Push task and two booleans: is_spam and is_unlearn */
+		struct rspamd_task **ptask = lua_newuserdata(L, sizeof(*ptask));
+		*ptask = task;
+		rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+		if (is_learn) {
+			lua_pushboolean(L, is_spam);
+			lua_pushboolean(L,
+							task->flags & RSPAMD_TASK_FLAG_UNLEARN ? true : false);
+			nargs = 3;
+		}
+		else {
+			nargs = 1;
+		}
+
+		if (lua_pcall(L, nargs, LUA_MULTRET, 0) != 0) {
+			msg_err_task("call to %s failed: %s",
+						 "condition callback",
+						 lua_tostring(L, -1));
+		}
+		else {
+			if (lua_isboolean(L, 1)) {
+				if (!lua_toboolean(L, 1)) {
+					ret = TRUE;
+				}
+			}
+
+			if (lua_isstring(L, 2)) {
+				if (ret) {
+					msg_notice_task("%s condition for classifier %s returned: %s; skip classifier",
+									is_learn ? "learn" : "classify", cl->cfg->name,
+									lua_tostring(L, 2));
+				}
+				else {
+					msg_info_task("%s condition for classifier %s returned: %s",
+								  is_learn ? "learn" : "classify", cl->cfg->name,
+								  lua_tostring(L, 2));
+				}
+			}
+			else if (ret) {
+				msg_notice_task("%s condition for classifier %s returned false; skip classifier",
+								is_learn ? "learn" : "classify", cl->cfg->name);
+			}
+
+			if (ret) {
+				lua_settop(L, old_top);
+				break;
+			}
+		}
+
+		lua_settop(L, old_top);
+		cur = g_list_next(cur);
+	}
+
+	return ret;
+}
+
+static void
+rspamd_stat_preprocess(struct rspamd_stat_ctx *st_ctx,
+					   struct rspamd_task *task, gboolean is_learn, gboolean is_spam)
+{
+	guint i;
+	struct rspamd_statfile *st;
+	gpointer bk_run;
+
+	if (task->tokens == NULL) {
+		rspamd_stat_process_tokenize(st_ctx, task);
+	}
+
+	task->stat_runtimes = g_ptr_array_sized_new(st_ctx->statfiles->len);
+	g_ptr_array_set_size(task->stat_runtimes, st_ctx->statfiles->len);
+	rspamd_mempool_add_destructor(task->task_pool,
+								  rspamd_ptr_array_free_hard, task->stat_runtimes);
+
+	/* Temporary set all stat_runtimes to some max size to distinguish from NULL */
+	for (i = 0; i < st_ctx->statfiles->len; i++) {
+		g_ptr_array_index(task->stat_runtimes, i) = GSIZE_TO_POINTER(G_MAXSIZE);
+	}
+
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		struct rspamd_classifier *cl = g_ptr_array_index(st_ctx->classifiers, i);
+		gboolean skip_classifier = FALSE;
+
+		if (cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND) {
+			skip_classifier = TRUE;
+		}
+		else {
+			if (rspamd_stat_classifier_is_skipped(task, cl, is_learn, is_spam)) {
+				skip_classifier = TRUE;
+			}
+		}
+
+		if (skip_classifier) {
+			/* Set NULL for all statfiles indexed by id */
+			for (int j = 0; j < cl->statfiles_ids->len; j++) {
+				int id = g_array_index(cl->statfiles_ids, gint, j);
+				g_ptr_array_index(task->stat_runtimes, id) = NULL;
+			}
+		}
+	}
+
+	for (i = 0; i < st_ctx->statfiles->len; i++) {
+		st = g_ptr_array_index(st_ctx->statfiles, i);
+		g_assert(st != NULL);
+
+		if (g_ptr_array_index(task->stat_runtimes, i) == NULL) {
+			/* The whole classifier is skipped */
+			continue;
+		}
+
+		if (is_learn && st->backend->read_only) {
+			/* Read only backend, skip it */
+			g_ptr_array_index(task->stat_runtimes, i) = NULL;
+			continue;
+		}
+
+		if (!is_learn && !rspamd_symcache_is_symbol_enabled(task, task->cfg->cache,
+															st->stcf->symbol)) {
+			g_ptr_array_index(task->stat_runtimes, i) = NULL;
+			msg_debug_bayes("symbol %s is disabled, skip classification",
+							st->stcf->symbol);
+			continue;
+		}
+
+		bk_run = st->backend->runtime(task, st->stcf, is_learn, st->bkcf, i);
+
+		if (bk_run == NULL) {
+			msg_err_task("cannot init backend %s for statfile %s",
+						 st->backend->name, st->stcf->symbol);
+		}
+
+		g_ptr_array_index(task->stat_runtimes, i) = bk_run;
+	}
+}
+
+static void
+rspamd_stat_backends_process(struct rspamd_stat_ctx *st_ctx,
+							 struct rspamd_task *task)
+{
+	guint i;
+	struct rspamd_statfile *st;
+	gpointer bk_run;
+
+	g_assert(task->stat_runtimes != NULL);
+
+	for (i = 0; i < st_ctx->statfiles->len; i++) {
+		st = g_ptr_array_index(st_ctx->statfiles, i);
+		bk_run = g_ptr_array_index(task->stat_runtimes, i);
+
+		if (bk_run != NULL) {
+			st->backend->process_tokens(task, task->tokens, i, bk_run);
+		}
+	}
+}
+
+static void
+rspamd_stat_classifiers_process(struct rspamd_stat_ctx *st_ctx,
+								struct rspamd_task *task)
+{
+	guint i, j, id;
+	struct rspamd_classifier *cl;
+	struct rspamd_statfile *st;
+	gpointer bk_run;
+	gboolean skip;
+
+	if (st_ctx->classifiers->len == 0) {
+		return;
+	}
+
+	/*
+	 * Do not classify a message if some class is missing
+	 */
+	if (!(task->flags & RSPAMD_TASK_FLAG_HAS_SPAM_TOKENS)) {
+		msg_info_task("skip statistics as SPAM class is missing");
+
+		return;
+	}
+	if (!(task->flags & RSPAMD_TASK_FLAG_HAS_HAM_TOKENS)) {
+		msg_info_task("skip statistics as HAM class is missing");
+
+		return;
+	}
+
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		cl = g_ptr_array_index(st_ctx->classifiers, i);
+		cl->spam_learns = 0;
+		cl->ham_learns = 0;
+	}
+
+	g_assert(task->stat_runtimes != NULL);
+
+	for (i = 0; i < st_ctx->statfiles->len; i++) {
+		st = g_ptr_array_index(st_ctx->statfiles, i);
+		cl = st->classifier;
+
+		bk_run = g_ptr_array_index(task->stat_runtimes, i);
+		g_assert(st != NULL);
+
+		if (bk_run != NULL) {
+			if (st->stcf->is_spam) {
+				cl->spam_learns += st->backend->total_learns(task,
+															 bk_run,
+															 st_ctx);
+			}
+			else {
+				cl->ham_learns += st->backend->total_learns(task,
+															bk_run,
+															st_ctx);
+			}
+		}
+	}
+
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+		g_assert(cl != NULL);
+
+		skip = FALSE;
+
+		/* Do not process classifiers on backend failures */
+		for (j = 0; j < cl->statfiles_ids->len; j++) {
+			id = g_array_index(cl->statfiles_ids, gint, j);
+			bk_run = g_ptr_array_index(task->stat_runtimes, id);
+			st = g_ptr_array_index(st_ctx->statfiles, id);
+
+			if (bk_run != NULL) {
+				if (!st->backend->finalize_process(task, bk_run, st_ctx)) {
+					skip = TRUE;
+					break;
+				}
+			}
+		}
+
+		/* Ensure that all symbols enabled */
+		if (!skip && !(cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND)) {
+			for (j = 0; j < cl->statfiles_ids->len; j++) {
+				id = g_array_index(cl->statfiles_ids, gint, j);
+				bk_run = g_ptr_array_index(task->stat_runtimes, id);
+				st = g_ptr_array_index(st_ctx->statfiles, id);
+
+				if (bk_run == NULL) {
+					skip = TRUE;
+					msg_debug_bayes("disable classifier %s as statfile symbol %s is disabled",
+									cl->cfg->name, st->stcf->symbol);
+					break;
+				}
+			}
+		}
+
+		if (!skip) {
+			if (cl->cfg->min_tokens > 0 && task->tokens->len < cl->cfg->min_tokens) {
+				msg_debug_bayes(
+					"contains less tokens than required for %s classifier: "
+					"%ud < %ud",
+					cl->cfg->name,
+					task->tokens->len,
+					cl->cfg->min_tokens);
+				continue;
+			}
+			else if (cl->cfg->max_tokens > 0 && task->tokens->len > cl->cfg->max_tokens) {
+				msg_debug_bayes(
+					"contains more tokens than allowed for %s classifier: "
+					"%ud > %ud",
+					cl->cfg->name,
+					task->tokens->len,
+					cl->cfg->max_tokens);
+				continue;
+			}
+
+			cl->subrs->classify_func(cl, task->tokens, task);
+		}
+	}
+}
+
+rspamd_stat_result_t
+rspamd_stat_classify(struct rspamd_task *task, lua_State *L, guint stage,
+					 GError **err)
+{
+	struct rspamd_stat_ctx *st_ctx;
+	rspamd_stat_result_t ret = RSPAMD_STAT_PROCESS_OK;
+
+	st_ctx = rspamd_stat_get_ctx();
+	g_assert(st_ctx != NULL);
+
+	if (st_ctx->classifiers->len == 0) {
+		task->processed_stages |= stage;
+		return ret;
+	}
+
+	if (stage == RSPAMD_TASK_STAGE_CLASSIFIERS_PRE) {
+		/* Preprocess tokens */
+		rspamd_stat_preprocess(st_ctx, task, FALSE, FALSE);
+	}
+	else if (stage == RSPAMD_TASK_STAGE_CLASSIFIERS) {
+		/* Process backends */
+		rspamd_stat_backends_process(st_ctx, task);
+	}
+	else if (stage == RSPAMD_TASK_STAGE_CLASSIFIERS_POST) {
+		/* Process classifiers */
+		rspamd_stat_classifiers_process(st_ctx, task);
+	}
+
+	task->processed_stages |= stage;
+
+	return ret;
+}
+
+static gboolean
+rspamd_stat_cache_check(struct rspamd_stat_ctx *st_ctx,
+						struct rspamd_task *task,
+						const gchar *classifier,
+						gboolean spam,
+						GError **err)
+{
+	rspamd_learn_t learn_res = RSPAMD_LEARN_OK;
+	struct rspamd_classifier *cl, *sel = NULL;
+	gpointer rt;
+	guint i;
+
+	/* Check whether we have learned that file */
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+		/* Skip other classifiers if they are not needed */
+		if (classifier != NULL && (cl->cfg->name == NULL ||
+								   g_ascii_strcasecmp(classifier, cl->cfg->name) != 0)) {
+			continue;
+		}
+
+		sel = cl;
+
+		if (sel->cache && sel->cachecf) {
+			rt = cl->cache->runtime(task, sel->cachecf, FALSE);
+			learn_res = cl->cache->check(task, spam, rt);
+		}
+
+		if (learn_res == RSPAMD_LEARN_IGNORE) {
+			/* Do not learn twice */
+			g_set_error(err, rspamd_stat_quark(), 404, "<%s> has been already "
+													   "learned as %s, ignore it",
+						MESSAGE_FIELD(task, message_id),
+						spam ? "spam" : "ham");
+			task->flags |= RSPAMD_TASK_FLAG_ALREADY_LEARNED;
+
+			return FALSE;
+		}
+		else if (learn_res == RSPAMD_LEARN_UNLEARN) {
+			task->flags |= RSPAMD_TASK_FLAG_UNLEARN;
+			break;
+		}
+	}
+
+	if (sel == NULL) {
+		if (classifier) {
+			g_set_error(err, rspamd_stat_quark(), 404, "cannot find classifier "
+													   "with name %s",
+						classifier);
+		}
+		else {
+			g_set_error(err, rspamd_stat_quark(), 404, "no classifiers defined");
+		}
+
+		return FALSE;
+	}
+
+	return TRUE;
+}
+
+static gboolean
+rspamd_stat_classifiers_learn(struct rspamd_stat_ctx *st_ctx,
+							  struct rspamd_task *task,
+							  const gchar *classifier,
+							  gboolean spam,
+							  GError **err)
+{
+	struct rspamd_classifier *cl, *sel = NULL;
+	guint i;
+	gboolean learned = FALSE, too_small = FALSE, too_large = FALSE;
+
+	if ((task->flags & RSPAMD_TASK_FLAG_ALREADY_LEARNED) && err != NULL &&
+		*err == NULL) {
+		/* Do not learn twice */
+		g_set_error(err, rspamd_stat_quark(), 208, "<%s> has been already "
+												   "learned as %s, ignore it",
+					MESSAGE_FIELD(task, message_id),
+					spam ? "spam" : "ham");
+
+		return FALSE;
+	}
+
+	/* Check whether we have learned that file */
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+		/* Skip other classifiers if they are not needed */
+		if (classifier != NULL && (cl->cfg->name == NULL ||
+								   g_ascii_strcasecmp(classifier, cl->cfg->name) != 0)) {
+			continue;
+		}
+
+		sel = cl;
+
+		/* Now check max and min tokens */
+		if (cl->cfg->min_tokens > 0 && task->tokens->len < cl->cfg->min_tokens) {
+			msg_info_task(
+				"<%s> contains less tokens than required for %s classifier: "
+				"%ud < %ud",
+				MESSAGE_FIELD(task, message_id),
+				cl->cfg->name,
+				task->tokens->len,
+				cl->cfg->min_tokens);
+			too_small = TRUE;
+			continue;
+		}
+		else if (cl->cfg->max_tokens > 0 && task->tokens->len > cl->cfg->max_tokens) {
+			msg_info_task(
+				"<%s> contains more tokens than allowed for %s classifier: "
+				"%ud > %ud",
+				MESSAGE_FIELD(task, message_id),
+				cl->cfg->name,
+				task->tokens->len,
+				cl->cfg->max_tokens);
+			too_large = TRUE;
+			continue;
+		}
+
+		if (cl->subrs->learn_spam_func(cl, task->tokens, task, spam,
+									   task->flags & RSPAMD_TASK_FLAG_UNLEARN, err)) {
+			learned = TRUE;
+		}
+	}
+
+	if (sel == NULL) {
+		if (classifier) {
+			g_set_error(err, rspamd_stat_quark(), 404, "cannot find classifier "
+													   "with name %s",
+						classifier);
+		}
+		else {
+			g_set_error(err, rspamd_stat_quark(), 404, "no classifiers defined");
+		}
+
+		return FALSE;
+	}
+
+	if (!learned && err && *err == NULL) {
+		if (too_large) {
+			g_set_error(err, rspamd_stat_quark(), 204,
+						"<%s> contains more tokens than allowed for %s classifier: "
+						"%d > %d",
+						MESSAGE_FIELD(task, message_id),
+						sel->cfg->name,
+						task->tokens->len,
+						sel->cfg->max_tokens);
+		}
+		else if (too_small) {
+			g_set_error(err, rspamd_stat_quark(), 204,
+						"<%s> contains less tokens than required for %s classifier: "
+						"%d < %d",
+						MESSAGE_FIELD(task, message_id),
+						sel->cfg->name,
+						task->tokens->len,
+						sel->cfg->min_tokens);
+		}
+	}
+
+	return learned;
+}
+
+static gboolean
+rspamd_stat_backends_learn(struct rspamd_stat_ctx *st_ctx,
+						   struct rspamd_task *task,
+						   const gchar *classifier,
+						   gboolean spam,
+						   GError **err)
+{
+	struct rspamd_classifier *cl, *sel = NULL;
+	struct rspamd_statfile *st;
+	gpointer bk_run;
+	guint i, j;
+	gint id;
+	gboolean res = FALSE, backend_found = FALSE;
+
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+		/* Skip other classifiers if they are not needed */
+		if (classifier != NULL && (cl->cfg->name == NULL ||
+								   g_ascii_strcasecmp(classifier, cl->cfg->name) != 0)) {
+			continue;
+		}
+
+		if (cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND) {
+			res = TRUE;
+			continue;
+		}
+
+		sel = cl;
+
+		for (j = 0; j < cl->statfiles_ids->len; j++) {
+			id = g_array_index(cl->statfiles_ids, gint, j);
+			st = g_ptr_array_index(st_ctx->statfiles, id);
+			bk_run = g_ptr_array_index(task->stat_runtimes, id);
+
+			g_assert(st != NULL);
+
+			if (bk_run == NULL) {
+				/* XXX: must be error */
+				if (task->result->passthrough_result) {
+					/* Passthrough email, cannot learn */
+					g_set_error(err, rspamd_stat_quark(), 204,
+								"Cannot learn statistics when passthrough "
+								"result has been set; not classified");
+
+					res = FALSE;
+					goto end;
+				}
+
+				msg_debug_task("no runtime for backend %s; classifier %s; symbol %s",
+							   st->backend->name, cl->cfg->name, st->stcf->symbol);
+				continue;
+			}
+
+			/* We set sel merely when we have runtime */
+			backend_found = TRUE;
+
+			if (!(task->flags & RSPAMD_TASK_FLAG_UNLEARN)) {
+				if (!!spam != !!st->stcf->is_spam) {
+					/* If we are not unlearning, then do not touch another class */
+					continue;
+				}
+			}
+
+			if (!st->backend->learn_tokens(task, task->tokens, id, bk_run)) {
+				g_set_error(err, rspamd_stat_quark(), 500,
+							"Cannot push "
+							"learned results to the backend");
+
+				res = FALSE;
+				goto end;
+			}
+			else {
+				if (!!spam == !!st->stcf->is_spam) {
+					st->backend->inc_learns(task, bk_run, st_ctx);
+				}
+				else if (task->flags & RSPAMD_TASK_FLAG_UNLEARN) {
+					st->backend->dec_learns(task, bk_run, st_ctx);
+				}
+
+				res = TRUE;
+			}
+		}
+	}
+
+end:
+
+	if (!res) {
+		if (err && *err) {
+			/* Error has been set already */
+			return res;
+		}
+
+		if (sel == NULL) {
+			if (classifier) {
+				g_set_error(err, rspamd_stat_quark(), 404, "cannot find classifier "
+														   "with name %s",
+							classifier);
+			}
+			else {
+				g_set_error(err, rspamd_stat_quark(), 404, "no classifiers defined");
+			}
+
+			return FALSE;
+		}
+		else if (!backend_found) {
+			g_set_error(err, rspamd_stat_quark(), 204, "all learn conditions "
+													   "denied learning %s in %s",
+						spam ? "spam" : "ham",
+						classifier ? classifier : "default classifier");
+		}
+		else {
+			g_set_error(err, rspamd_stat_quark(), 404, "cannot find statfile "
+													   "backend to learn %s in %s",
+						spam ? "spam" : "ham",
+						classifier ? classifier : "default classifier");
+		}
+	}
+
+	return res;
+}
+
+static gboolean
+rspamd_stat_backends_post_learn(struct rspamd_stat_ctx *st_ctx,
+								struct rspamd_task *task,
+								const gchar *classifier,
+								gboolean spam,
+								GError **err)
+{
+	struct rspamd_classifier *cl;
+	struct rspamd_statfile *st;
+	gpointer bk_run, cache_run;
+	guint i, j;
+	gint id;
+	gboolean res = TRUE;
+
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+		/* Skip other classifiers if they are not needed */
+		if (classifier != NULL && (cl->cfg->name == NULL ||
+								   g_ascii_strcasecmp(classifier, cl->cfg->name) != 0)) {
+			continue;
+		}
+
+		if (cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND) {
+			res = TRUE;
+			continue;
+		}
+
+		for (j = 0; j < cl->statfiles_ids->len; j++) {
+			id = g_array_index(cl->statfiles_ids, gint, j);
+			st = g_ptr_array_index(st_ctx->statfiles, id);
+			bk_run = g_ptr_array_index(task->stat_runtimes, id);
+
+			g_assert(st != NULL);
+
+			if (bk_run == NULL) {
+				/* XXX: must be error */
+				continue;
+			}
+
+			if (!st->backend->finalize_learn(task, bk_run, st_ctx, err)) {
+				return RSPAMD_STAT_PROCESS_ERROR;
+			}
+		}
+
+		if (cl->cache) {
+			cache_run = cl->cache->runtime(task, cl->cachecf, TRUE);
+			cl->cache->learn(task, spam, cache_run);
+		}
+	}
+
+	g_atomic_int_add(&task->worker->srv->stat->messages_learned, 1);
+
+	return res;
+}
+
+rspamd_stat_result_t
+rspamd_stat_learn(struct rspamd_task *task,
+				  gboolean spam, lua_State *L, const gchar *classifier, guint stage,
+				  GError **err)
+{
+	struct rspamd_stat_ctx *st_ctx;
+	rspamd_stat_result_t ret = RSPAMD_STAT_PROCESS_OK;
+
+	/*
+	 * We assume now that a task has been already classified before
+	 * coming to learn
+	 */
+	g_assert(RSPAMD_TASK_IS_CLASSIFIED(task));
+
+	st_ctx = rspamd_stat_get_ctx();
+	g_assert(st_ctx != NULL);
+
+	if (st_ctx->classifiers->len == 0) {
+		task->processed_stages |= stage;
+		return ret;
+	}
+
+	if (stage == RSPAMD_TASK_STAGE_LEARN_PRE) {
+		/* Process classifiers */
+		rspamd_stat_preprocess(st_ctx, task, TRUE, spam);
+
+		if (!rspamd_stat_cache_check(st_ctx, task, classifier, spam, err)) {
+			return RSPAMD_STAT_PROCESS_ERROR;
+		}
+	}
+	else if (stage == RSPAMD_TASK_STAGE_LEARN) {
+		/* Process classifiers */
+		if (!rspamd_stat_classifiers_learn(st_ctx, task, classifier,
+										   spam, err)) {
+			if (err && *err == NULL) {
+				g_set_error(err, rspamd_stat_quark(), 500,
+							"Unknown statistics error, found when learning classifiers;"
+							" classifier: %s",
+							task->classifier);
+			}
+			return RSPAMD_STAT_PROCESS_ERROR;
+		}
+
+		/* Process backends */
+		if (!rspamd_stat_backends_learn(st_ctx, task, classifier, spam, err)) {
+			if (err && *err == NULL) {
+				g_set_error(err, rspamd_stat_quark(), 500,
+							"Unknown statistics error, found when storing data on backend;"
+							" classifier: %s",
+							task->classifier);
+			}
+			return RSPAMD_STAT_PROCESS_ERROR;
+		}
+	}
+	else if (stage == RSPAMD_TASK_STAGE_LEARN_POST) {
+		if (!rspamd_stat_backends_post_learn(st_ctx, task, classifier, spam, err)) {
+			return RSPAMD_STAT_PROCESS_ERROR;
+		}
+	}
+
+	task->processed_stages |= stage;
+
+	return ret;
+}
+
+static gboolean
+rspamd_stat_has_classifier_symbols(struct rspamd_task *task,
+								   struct rspamd_scan_result *mres,
+								   struct rspamd_classifier *cl)
+{
+	guint i;
+	gint id;
+	struct rspamd_statfile *st;
+	struct rspamd_stat_ctx *st_ctx;
+	gboolean is_spam;
+
+	if (mres == NULL) {
+		return FALSE;
+	}
+
+	st_ctx = rspamd_stat_get_ctx();
+	is_spam = !!(task->flags & RSPAMD_TASK_FLAG_LEARN_SPAM);
+
+	for (i = 0; i < cl->statfiles_ids->len; i++) {
+		id = g_array_index(cl->statfiles_ids, gint, i);
+		st = g_ptr_array_index(st_ctx->statfiles, id);
+
+		if (rspamd_task_find_symbol_result(task, st->stcf->symbol, NULL)) {
+			if (is_spam == !!st->stcf->is_spam) {
+				msg_debug_bayes("do not autolearn %s as symbol %s is already "
+								"added",
+								is_spam ? "spam" : "ham", st->stcf->symbol);
+
+				return TRUE;
+			}
+		}
+	}
+
+	return FALSE;
+}
+
+gboolean
+rspamd_stat_check_autolearn(struct rspamd_task *task)
+{
+	struct rspamd_stat_ctx *st_ctx;
+	struct rspamd_classifier *cl;
+	const ucl_object_t *obj, *elt1, *elt2;
+	struct rspamd_scan_result *mres = NULL;
+	struct rspamd_task **ptask;
+	lua_State *L;
+	guint i;
+	gint err_idx;
+	gboolean ret = FALSE;
+	gdouble ham_score, spam_score;
+	const gchar *lua_script, *lua_ret;
+
+	g_assert(RSPAMD_TASK_IS_CLASSIFIED(task));
+	st_ctx = rspamd_stat_get_ctx();
+	g_assert(st_ctx != NULL);
+
+	L = task->cfg->lua_state;
+
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		cl = g_ptr_array_index(st_ctx->classifiers, i);
+		ret = FALSE;
+
+		if (cl->cfg->opts) {
+			obj = ucl_object_lookup(cl->cfg->opts, "autolearn");
+
+			if (ucl_object_type(obj) == UCL_BOOLEAN) {
+				/* Legacy true/false */
+				if (ucl_object_toboolean(obj)) {
+					/*
+					 * Default learning algorithm:
+					 *
+					 * - We learn spam if action is ACTION_REJECT
+					 * - We learn ham if score is less than zero
+					 */
+					mres = task->result;
+
+					if (mres) {
+						if (mres->score > rspamd_task_get_required_score(task, mres)) {
+							task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+
+							ret = TRUE;
+						}
+						else if (mres->score < 0) {
+							task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+							ret = TRUE;
+						}
+					}
+				}
+			}
+			else if (ucl_object_type(obj) == UCL_ARRAY && obj->len == 2) {
+				/* Legacy thresholds */
+				/*
+				 * We have an array of 2 elements, treat it as a
+				 * ham_score, spam_score
+				 */
+				elt1 = ucl_array_find_index(obj, 0);
+				elt2 = ucl_array_find_index(obj, 1);
+
+				if ((ucl_object_type(elt1) == UCL_FLOAT ||
+					 ucl_object_type(elt1) == UCL_INT) &&
+					(ucl_object_type(elt2) == UCL_FLOAT ||
+					 ucl_object_type(elt2) == UCL_INT)) {
+					ham_score = ucl_object_todouble(elt1);
+					spam_score = ucl_object_todouble(elt2);
+
+					if (ham_score > spam_score) {
+						gdouble t;
+
+						t = ham_score;
+						ham_score = spam_score;
+						spam_score = t;
+					}
+
+					mres = task->result;
+
+					if (mres) {
+						if (mres->score >= spam_score) {
+							task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+
+							ret = TRUE;
+						}
+						else if (mres->score <= ham_score) {
+							task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+							ret = TRUE;
+						}
+					}
+				}
+			}
+			else if (ucl_object_type(obj) == UCL_STRING) {
+				/* Legacy script */
+				lua_script = ucl_object_tostring(obj);
+
+				if (luaL_dostring(L, lua_script) != 0) {
+					msg_err_task("cannot execute lua script for autolearn "
+								 "extraction: %s",
+								 lua_tostring(L, -1));
+				}
+				else {
+					if (lua_type(L, -1) == LUA_TFUNCTION) {
+						lua_pushcfunction(L, &rspamd_lua_traceback);
+						err_idx = lua_gettop(L);
+						lua_pushvalue(L, -2); /* Function itself */
+
+						ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+						*ptask = task;
+						rspamd_lua_setclass(L, "rspamd{task}", -1);
+
+						if (lua_pcall(L, 1, 1, err_idx) != 0) {
+							msg_err_task("call to autolearn script failed: "
+										 "%s",
+										 lua_tostring(L, -1));
+						}
+						else {
+							lua_ret = lua_tostring(L, -1);
+
+							/* We can have immediate results */
+							if (lua_ret) {
+								if (strcmp(lua_ret, "ham") == 0) {
+									task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+									ret = TRUE;
+								}
+								else if (strcmp(lua_ret, "spam") == 0) {
+									task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+									ret = TRUE;
+								}
+							}
+						}
+
+						/* Result + error function + original function */
+						lua_pop(L, 3);
+					}
+					else {
+						msg_err_task("lua script must return "
+									 "function(task) and not %s",
+									 lua_typename(L, lua_type(
+														 L, -1)));
+					}
+				}
+			}
+			else if (ucl_object_type(obj) == UCL_OBJECT) {
+				/* Try to find autolearn callback */
+				if (cl->autolearn_cbref == 0) {
+					/* We don't have preprocessed cb id, so try to get it */
+					if (!rspamd_lua_require_function(L, "lua_bayes_learn",
+													 "autolearn")) {
+						msg_err_task("cannot get autolearn library from "
+									 "`lua_bayes_learn`");
+					}
+					else {
+						cl->autolearn_cbref = luaL_ref(L, LUA_REGISTRYINDEX);
+					}
+				}
+
+				if (cl->autolearn_cbref != -1) {
+					lua_pushcfunction(L, &rspamd_lua_traceback);
+					err_idx = lua_gettop(L);
+					lua_rawgeti(L, LUA_REGISTRYINDEX, cl->autolearn_cbref);
+
+					ptask = lua_newuserdata(L, sizeof(struct rspamd_task *));
+					*ptask = task;
+					rspamd_lua_setclass(L, "rspamd{task}", -1);
+					/* Push the whole object as well */
+					ucl_object_push_lua(L, obj, true);
+
+					if (lua_pcall(L, 2, 1, err_idx) != 0) {
+						msg_err_task("call to autolearn script failed: "
+									 "%s",
+									 lua_tostring(L, -1));
+					}
+					else {
+						lua_ret = lua_tostring(L, -1);
+
+						if (lua_ret) {
+							if (strcmp(lua_ret, "ham") == 0) {
+								task->flags |= RSPAMD_TASK_FLAG_LEARN_HAM;
+								ret = TRUE;
+							}
+							else if (strcmp(lua_ret, "spam") == 0) {
+								task->flags |= RSPAMD_TASK_FLAG_LEARN_SPAM;
+								ret = TRUE;
+							}
+						}
+					}
+
+					lua_settop(L, err_idx - 1);
+				}
+			}
+
+			if (ret) {
+				/* Do not autolearn if we have this symbol already */
+				if (rspamd_stat_has_classifier_symbols(task, mres, cl)) {
+					ret = FALSE;
+					task->flags &= ~(RSPAMD_TASK_FLAG_LEARN_HAM |
+									 RSPAMD_TASK_FLAG_LEARN_SPAM);
+				}
+				else if (mres != NULL) {
+					if (task->flags & RSPAMD_TASK_FLAG_LEARN_HAM) {
+						msg_info_task("<%s>: autolearn ham for classifier "
+									  "'%s' as message's "
+									  "score is negative: %.2f",
+									  MESSAGE_FIELD(task, message_id), cl->cfg->name,
+									  mres->score);
+					}
+					else {
+						msg_info_task("<%s>: autolearn spam for classifier "
+									  "'%s' as message's "
+									  "action is reject, score: %.2f",
+									  MESSAGE_FIELD(task, message_id), cl->cfg->name,
+									  mres->score);
+					}
+
+					task->classifier = cl->cfg->name;
+					break;
+				}
+			}
+		}
+	}
+
+	return ret;
+}
+
+/**
+ * Get the overall statistics for all statfile backends
+ * @param cfg configuration
+ * @param total_learns the total number of learns is stored here
+ * @return array of statistical information
+ */
+rspamd_stat_result_t
+rspamd_stat_statistics(struct rspamd_task *task,
+					   struct rspamd_config *cfg,
+					   guint64 *total_learns,
+					   ucl_object_t **target)
+{
+	struct rspamd_stat_ctx *st_ctx;
+	struct rspamd_classifier *cl;
+	struct rspamd_statfile *st;
+	gpointer backend_runtime;
+	ucl_object_t *res = NULL, *elt;
+	guint64 learns = 0;
+	guint i, j;
+	gint id;
+
+	st_ctx = rspamd_stat_get_ctx();
+	g_assert(st_ctx != NULL);
+
+	res = ucl_object_typed_new(UCL_ARRAY);
+
+	for (i = 0; i < st_ctx->classifiers->len; i++) {
+		cl = g_ptr_array_index(st_ctx->classifiers, i);
+
+		if (cl->cfg->flags & RSPAMD_FLAG_CLASSIFIER_NO_BACKEND) {
+			continue;
+		}
+
+		for (j = 0; j < cl->statfiles_ids->len; j++) {
+			id = g_array_index(cl->statfiles_ids, gint, j);
+			st = g_ptr_array_index(st_ctx->statfiles, id);
+			backend_runtime = st->backend->runtime(task, st->stcf, FALSE,
+												   st->bkcf, id);
+			elt = st->backend->get_stat(backend_runtime, st->bkcf);
+
+			if (elt && ucl_object_type(elt) == UCL_OBJECT) {
+				const ucl_object_t *rev = ucl_object_lookup(elt, "revision");
+
+				learns += ucl_object_toint(rev);
+			}
+			else {
+				learns += st->backend->total_learns(task, backend_runtime,
+													st->bkcf);
+			}
+
+			if (elt != NULL) {
+				ucl_array_append(res, elt);
+			}
+		}
+	}
+
+	if (total_learns != NULL) {
+		*total_learns = learns;
+	}
+
+	if (target) {
+		*target = res;
+	}
+	else {
+		ucl_object_unref(res);
+	}
+
+	return RSPAMD_STAT_PROCESS_OK;
+}