Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 2122 insertions, 0 deletions

diff --git a/src/plugins/chartable.cxx b/src/plugins/chartable.cxx
new file mode 100644
index 0000000..704f12a
--- /dev/null
+++ b/src/plugins/chartable.cxx
@@ -0,0 +1,2122 @@
+/*-
+ * Copyright 2016 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/***MODULE:chartable
+ * rspamd module that make marks based on symbol chains
+ *
+ * Allowed options:
+ * - symbol (string): symbol to insert (default: 'R_BAD_CHARSET')
+ * - threshold (double): value that would be used as threshold in expression characters_changed / total_characters
+ *   (e.g. if threshold is 0.1 than charset change should occur more often than in 10 symbols), default: 0.1
+ */
+
+
+#include "config.h"
+#include "libmime/message.h"
+#include "rspamd.h"
+#include "libstat/stat_api.h"
+#include "libmime/lang_detection.h"
+
+#include "unicode/utf8.h"
+#include "unicode/uchar.h"
+#include "contrib/ankerl/unordered_dense.h"
+
+#define DEFAULT_SYMBOL "R_MIXED_CHARSET"
+#define DEFAULT_URL_SYMBOL "R_MIXED_CHARSET_URL"
+#define DEFAULT_THRESHOLD 0.1
+
+#define msg_debug_chartable(...) rspamd_conditional_debug_fast(nullptr, task->from_addr,                                       \
+															   rspamd_chartable_log_id, "chartable", task->task_pool->tag.uid, \
+															   G_STRFUNC,                                                      \
+															   __VA_ARGS__)
+
+INIT_LOG_MODULE(chartable)
+
+/* Initialization */
+gint chartable_module_init(struct rspamd_config *cfg, struct module_ctx **ctx);
+
+gint chartable_module_config(struct rspamd_config *cfg, bool validate);
+
+gint chartable_module_reconfig(struct rspamd_config *cfg);
+
+module_t chartable_module = {
+	"chartable",
+	chartable_module_init,
+	chartable_module_config,
+	chartable_module_reconfig,
+	nullptr,
+	RSPAMD_MODULE_VER,
+	(guint) -1,
+};
+
+struct chartable_ctx {
+	struct module_ctx ctx;
+	const gchar *symbol;
+	const gchar *url_symbol;
+	double threshold;
+	guint max_word_len;
+};
+
+static inline struct chartable_ctx *
+chartable_get_context(struct rspamd_config *cfg)
+{
+	return (struct chartable_ctx *) g_ptr_array_index(cfg->c_modules,
+													  chartable_module.ctx_offset);
+}
+
+static void chartable_symbol_callback(struct rspamd_task *task,
+									  struct rspamd_symcache_dynamic_item *item,
+									  void *unused);
+
+static void chartable_url_symbol_callback(struct rspamd_task *task,
+										  struct rspamd_symcache_dynamic_item *item,
+										  void *unused);
+
+gint chartable_module_init(struct rspamd_config *cfg, struct module_ctx **ctx)
+{
+	struct chartable_ctx *chartable_module_ctx;
+
+	chartable_module_ctx = rspamd_mempool_alloc0_type(cfg->cfg_pool,
+													  struct chartable_ctx);
+	chartable_module_ctx->max_word_len = 10;
+
+	*ctx = (struct module_ctx *) chartable_module_ctx;
+
+	return 0;
+}
+
+
+gint chartable_module_config(struct rspamd_config *cfg, bool _)
+{
+	const ucl_object_t *value;
+	gint res = TRUE;
+	struct chartable_ctx *chartable_module_ctx = chartable_get_context(cfg);
+
+	if (!rspamd_config_is_module_enabled(cfg, "chartable")) {
+		return TRUE;
+	}
+
+	if ((value =
+			 rspamd_config_get_module_opt(cfg, "chartable", "symbol")) != nullptr) {
+		chartable_module_ctx->symbol = ucl_obj_tostring(value);
+	}
+	else {
+		chartable_module_ctx->symbol = DEFAULT_SYMBOL;
+	}
+	if ((value =
+			 rspamd_config_get_module_opt(cfg, "chartable", "url_symbol")) != nullptr) {
+		chartable_module_ctx->url_symbol = ucl_obj_tostring(value);
+	}
+	else {
+		chartable_module_ctx->url_symbol = DEFAULT_URL_SYMBOL;
+	}
+	if ((value =
+			 rspamd_config_get_module_opt(cfg, "chartable", "threshold")) != nullptr) {
+		if (!ucl_obj_todouble_safe(value, &chartable_module_ctx->threshold)) {
+			msg_warn_config("invalid numeric value");
+			chartable_module_ctx->threshold = DEFAULT_THRESHOLD;
+		}
+	}
+	else {
+		chartable_module_ctx->threshold = DEFAULT_THRESHOLD;
+	}
+	if ((value =
+			 rspamd_config_get_module_opt(cfg, "chartable", "max_word_len")) != nullptr) {
+		chartable_module_ctx->max_word_len = ucl_object_toint(value);
+	}
+	else {
+		chartable_module_ctx->threshold = DEFAULT_THRESHOLD;
+	}
+
+	rspamd_symcache_add_symbol(cfg->cache,
+							   chartable_module_ctx->symbol,
+							   0,
+							   chartable_symbol_callback,
+							   nullptr,
+							   SYMBOL_TYPE_NORMAL,
+							   -1);
+	rspamd_symcache_add_symbol(cfg->cache,
+							   chartable_module_ctx->url_symbol,
+							   0,
+							   chartable_url_symbol_callback,
+							   nullptr,
+							   SYMBOL_TYPE_NORMAL,
+							   -1);
+
+	msg_info_config("init internal chartable module");
+
+	return res;
+}
+
+gint chartable_module_reconfig(struct rspamd_config *cfg)
+{
+	return chartable_module_config(cfg, false);
+}
+
+static const auto latin_confusable = ankerl::unordered_dense::set<int>{
+	0x02028,
+	0x02029,
+	0x01680,
+	0x02000,
+	0x02001,
+	0x02002,
+	0x02003,
+	0x02004,
+	0x02005,
+	0x02006,
+	0x02008,
+	0x02009,
+	0x0200a,
+	0x0205f,
+	0x000a0,
+	0x02007,
+	0x0202f,
+	0x007fa,
+	0x0fe4d,
+	0x0fe4e,
+	0x0fe4f,
+	0x02010,
+	0x02011,
+	0x02012,
+	0x02013,
+	0x0fe58,
+	0x006d4,
+	0x02043,
+	0x002d7,
+	0x02212,
+	0x02796,
+	0x02cba,
+	0x0060d,
+	0x0066b,
+	0x0201a,
+	0x000b8,
+	0x0a4f9,
+	0x0037e,
+	0x00903,
+	0x00a83,
+	0x0ff1a,
+	0x00589,
+	0x00703,
+	0x00704,
+	0x016ec,
+	0x0fe30,
+	0x01803,
+	0x01809,
+	0x0205a,
+	0x005c3,
+	0x002f8,
+	0x0a789,
+	0x02236,
+	0x002d0,
+	0x0a4fd,
+	0x0ff01,
+	0x001c3,
+	0x02d51,
+	0x00294,
+	0x00241,
+	0x0097d,
+	0x013ae,
+	0x0a6eb,
+	0x1d16d,
+	0x02024,
+	0x00701,
+	0x00702,
+	0x0a60e,
+	0x10a50,
+	0x00660,
+	0x006f0,
+	0x0a4f8,
+	0x0055d,
+	0x0ff07,
+	0x02018,
+	0x02019,
+	0x0201b,
+	0x02032,
+	0x02035,
+	0x0055a,
+	0x005f3,
+	0x00060,
+	0x01fef,
+	0x0ff40,
+	0x000b4,
+	0x00384,
+	0x01ffd,
+	0x01fbd,
+	0x01fbf,
+	0x01ffe,
+	0x002b9,
+	0x00374,
+	0x002c8,
+	0x002ca,
+	0x002cb,
+	0x002f4,
+	0x002bb,
+	0x002bd,
+	0x002bc,
+	0x002be,
+	0x0a78c,
+	0x005d9,
+	0x007f4,
+	0x007f5,
+	0x0144a,
+	0x016cc,
+	0x16f51,
+	0x16f52,
+	0x0ff3b,
+	0x02768,
+	0x02772,
+	0x03014,
+	0x0fd3e,
+	0x0ff3d,
+	0x02769,
+	0x02773,
+	0x03015,
+	0x0fd3f,
+	0x02774,
+	0x1d114,
+	0x02775,
+	0x0204e,
+	0x0066d,
+	0x02217,
+	0x1031f,
+	0x01735,
+	0x02041,
+	0x02215,
+	0x02044,
+	0x02571,
+	0x027cb,
+	0x029f8,
+	0x1d23a,
+	0x031d3,
+	0x03033,
+	0x02cc6,
+	0x030ce,
+	0x04e3f,
+	0x02f03,
+	0x0ff3c,
+	0x0fe68,
+	0x02216,
+	0x027cd,
+	0x029f5,
+	0x029f9,
+	0x1d20f,
+	0x1d23b,
+	0x031d4,
+	0x04e36,
+	0x02f02,
+	0x0a778,
+	0x002c4,
+	0x002c6,
+	0x016ed,
+	0x02795,
+	0x1029b,
+	0x02039,
+	0x0276e,
+	0x002c2,
+	0x1d236,
+	0x01438,
+	0x016b2,
+	0x01400,
+	0x02e40,
+	0x030a0,
+	0x0a4ff,
+	0x0203a,
+	0x0276f,
+	0x002c3,
+	0x1d237,
+	0x01433,
+	0x16f3f,
+	0x02053,
+	0x002dc,
+	0x01fc0,
+	0x0223c,
+	0x1d7d0,
+	0x1d7da,
+	0x1d7e4,
+	0x1d7ee,
+	0x1d7f8,
+	0x0a75a,
+	0x001a7,
+	0x003e8,
+	0x0a644,
+	0x014bf,
+	0x0a6ef,
+	0x1d206,
+	0x1d7d1,
+	0x1d7db,
+	0x1d7e5,
+	0x1d7ef,
+	0x1d7f9,
+	0x0a7ab,
+	0x0021c,
+	0x001b7,
+	0x0a76a,
+	0x02ccc,
+	0x00417,
+	0x004e0,
+	0x16f3b,
+	0x118ca,
+	0x1d7d2,
+	0x1d7dc,
+	0x1d7e6,
+	0x1d7f0,
+	0x1d7fa,
+	0x013ce,
+	0x118af,
+	0x1d7d3,
+	0x1d7dd,
+	0x1d7e7,
+	0x1d7f1,
+	0x1d7fb,
+	0x001bc,
+	0x118bb,
+	0x1d7d4,
+	0x1d7de,
+	0x1d7e8,
+	0x1d7f2,
+	0x1d7fc,
+	0x02cd2,
+	0x00431,
+	0x013ee,
+	0x118d5,
+	0x1d212,
+	0x1d7d5,
+	0x1d7df,
+	0x1d7e9,
+	0x1d7f3,
+	0x1d7fd,
+	0x104d2,
+	0x118c6,
+	0x00b03,
+	0x009ea,
+	0x00a6a,
+	0x1e8cb,
+	0x1d7d6,
+	0x1d7e0,
+	0x1d7ea,
+	0x1d7f4,
+	0x1d7fe,
+	0x00223,
+	0x00222,
+	0x1031a,
+	0x00a67,
+	0x00b68,
+	0x009ed,
+	0x00d6d,
+	0x1d7d7,
+	0x1d7e1,
+	0x1d7eb,
+	0x1d7f5,
+	0x1d7ff,
+	0x0a76e,
+	0x02cca,
+	0x118cc,
+	0x118ac,
+	0x118d6,
+	0x0237a,
+	0x0ff41,
+	0x1d41a,
+	0x1d44e,
+	0x1d482,
+	0x1d4b6,
+	0x1d4ea,
+	0x1d51e,
+	0x1d552,
+	0x1d586,
+	0x1d5ba,
+	0x1d5ee,
+	0x1d622,
+	0x1d656,
+	0x1d68a,
+	0x00251,
+	0x003b1,
+	0x1d6c2,
+	0x1d6fc,
+	0x1d736,
+	0x1d770,
+	0x1d7aa,
+	0x00430,
+	0x0ff21,
+	0x1d400,
+	0x1d434,
+	0x1d468,
+	0x1d49c,
+	0x1d4d0,
+	0x1d504,
+	0x1d538,
+	0x1d56c,
+	0x1d5a0,
+	0x1d5d4,
+	0x1d608,
+	0x1d63c,
+	0x1d670,
+	0x00391,
+	0x1d6a8,
+	0x1d6e2,
+	0x1d71c,
+	0x1d756,
+	0x1d790,
+	0x00410,
+	0x013aa,
+	0x015c5,
+	0x0a4ee,
+	0x16f40,
+	0x102a0,
+	0x1d41b,
+	0x1d44f,
+	0x1d483,
+	0x1d4b7,
+	0x1d4eb,
+	0x1d51f,
+	0x1d553,
+	0x1d587,
+	0x1d5bb,
+	0x1d5ef,
+	0x1d623,
+	0x1d657,
+	0x1d68b,
+	0x00184,
+	0x0042c,
+	0x013cf,
+	0x015af,
+	0x0ff22,
+	0x0212c,
+	0x1d401,
+	0x1d435,
+	0x1d469,
+	0x1d4d1,
+	0x1d505,
+	0x1d539,
+	0x1d56d,
+	0x1d5a1,
+	0x1d5d5,
+	0x1d609,
+	0x1d63d,
+	0x1d671,
+	0x0a7b4,
+	0x00392,
+	0x1d6a9,
+	0x1d6e3,
+	0x1d71d,
+	0x1d757,
+	0x1d791,
+	0x00412,
+	0x013f4,
+	0x015f7,
+	0x0a4d0,
+	0x10282,
+	0x102a1,
+	0x10301,
+	0x0ff43,
+	0x0217d,
+	0x1d41c,
+	0x1d450,
+	0x1d484,
+	0x1d4b8,
+	0x1d4ec,
+	0x1d520,
+	0x1d554,
+	0x1d588,
+	0x1d5bc,
+	0x1d5f0,
+	0x1d624,
+	0x1d658,
+	0x1d68c,
+	0x01d04,
+	0x003f2,
+	0x02ca5,
+	0x00441,
+	0x0abaf,
+	0x1043d,
+	0x1f74c,
+	0x118f2,
+	0x118e9,
+	0x0ff23,
+	0x0216d,
+	0x02102,
+	0x0212d,
+	0x1d402,
+	0x1d436,
+	0x1d46a,
+	0x1d49e,
+	0x1d4d2,
+	0x1d56e,
+	0x1d5a2,
+	0x1d5d6,
+	0x1d60a,
+	0x1d63e,
+	0x1d672,
+	0x003f9,
+	0x02ca4,
+	0x00421,
+	0x013df,
+	0x0a4da,
+	0x102a2,
+	0x10302,
+	0x10415,
+	0x1051c,
+	0x0217e,
+	0x02146,
+	0x1d41d,
+	0x1d451,
+	0x1d485,
+	0x1d4b9,
+	0x1d4ed,
+	0x1d521,
+	0x1d555,
+	0x1d589,
+	0x1d5bd,
+	0x1d5f1,
+	0x1d625,
+	0x1d659,
+	0x1d68d,
+	0x00501,
+	0x013e7,
+	0x0146f,
+	0x0a4d2,
+	0x0216e,
+	0x02145,
+	0x1d403,
+	0x1d437,
+	0x1d46b,
+	0x1d49f,
+	0x1d4d3,
+	0x1d507,
+	0x1d53b,
+	0x1d56f,
+	0x1d5a3,
+	0x1d5d7,
+	0x1d60b,
+	0x1d63f,
+	0x1d673,
+	0x013a0,
+	0x015de,
+	0x015ea,
+	0x0a4d3,
+	0x0212e,
+	0x0ff45,
+	0x0212f,
+	0x02147,
+	0x1d41e,
+	0x1d452,
+	0x1d486,
+	0x1d4ee,
+	0x1d522,
+	0x1d556,
+	0x1d58a,
+	0x1d5be,
+	0x1d5f2,
+	0x1d626,
+	0x1d65a,
+	0x1d68e,
+	0x0ab32,
+	0x00435,
+	0x004bd,
+	0x022ff,
+	0x0ff25,
+	0x02130,
+	0x1d404,
+	0x1d438,
+	0x1d46c,
+	0x1d4d4,
+	0x1d508,
+	0x1d53c,
+	0x1d570,
+	0x1d5a4,
+	0x1d5d8,
+	0x1d60c,
+	0x1d640,
+	0x1d674,
+	0x00395,
+	0x1d6ac,
+	0x1d6e6,
+	0x1d720,
+	0x1d75a,
+	0x1d794,
+	0x00415,
+	0x02d39,
+	0x013ac,
+	0x0a4f0,
+	0x118a6,
+	0x118ae,
+	0x10286,
+	0x1d41f,
+	0x1d453,
+	0x1d487,
+	0x1d4bb,
+	0x1d4ef,
+	0x1d523,
+	0x1d557,
+	0x1d58b,
+	0x1d5bf,
+	0x1d5f3,
+	0x1d627,
+	0x1d65b,
+	0x1d68f,
+	0x0ab35,
+	0x0a799,
+	0x0017f,
+	0x01e9d,
+	0x00584,
+	0x1d213,
+	0x02131,
+	0x1d405,
+	0x1d439,
+	0x1d46d,
+	0x1d4d5,
+	0x1d509,
+	0x1d53d,
+	0x1d571,
+	0x1d5a5,
+	0x1d5d9,
+	0x1d60d,
+	0x1d641,
+	0x1d675,
+	0x0a798,
+	0x003dc,
+	0x1d7ca,
+	0x015b4,
+	0x0a4dd,
+	0x118c2,
+	0x118a2,
+	0x10287,
+	0x102a5,
+	0x10525,
+	0x0ff47,
+	0x0210a,
+	0x1d420,
+	0x1d454,
+	0x1d488,
+	0x1d4f0,
+	0x1d524,
+	0x1d558,
+	0x1d58c,
+	0x1d5c0,
+	0x1d5f4,
+	0x1d628,
+	0x1d65c,
+	0x1d690,
+	0x00261,
+	0x01d83,
+	0x0018d,
+	0x00581,
+	0x1d406,
+	0x1d43a,
+	0x1d46e,
+	0x1d4a2,
+	0x1d4d6,
+	0x1d50a,
+	0x1d53e,
+	0x1d572,
+	0x1d5a6,
+	0x1d5da,
+	0x1d60e,
+	0x1d642,
+	0x1d676,
+	0x0050c,
+	0x013c0,
+	0x013f3,
+	0x0a4d6,
+	0x0ff48,
+	0x0210e,
+	0x1d421,
+	0x1d489,
+	0x1d4bd,
+	0x1d4f1,
+	0x1d525,
+	0x1d559,
+	0x1d58d,
+	0x1d5c1,
+	0x1d5f5,
+	0x1d629,
+	0x1d65d,
+	0x1d691,
+	0x004bb,
+	0x00570,
+	0x013c2,
+	0x0ff28,
+	0x0210b,
+	0x0210c,
+	0x0210d,
+	0x1d407,
+	0x1d43b,
+	0x1d46f,
+	0x1d4d7,
+	0x1d573,
+	0x1d5a7,
+	0x1d5db,
+	0x1d60f,
+	0x1d643,
+	0x1d677,
+	0x00397,
+	0x1d6ae,
+	0x1d6e8,
+	0x1d722,
+	0x1d75c,
+	0x1d796,
+	0x02c8e,
+	0x0041d,
+	0x013bb,
+	0x0157c,
+	0x0a4e7,
+	0x102cf,
+	0x002db,
+	0x02373,
+	0x0ff49,
+	0x02170,
+	0x02139,
+	0x02148,
+	0x1d422,
+	0x1d456,
+	0x1d48a,
+	0x1d4be,
+	0x1d4f2,
+	0x1d526,
+	0x1d55a,
+	0x1d58e,
+	0x1d5c2,
+	0x1d5f6,
+	0x1d62a,
+	0x1d65e,
+	0x1d692,
+	0x00131,
+	0x1d6a4,
+	0x0026a,
+	0x00269,
+	0x003b9,
+	0x01fbe,
+	0x0037a,
+	0x1d6ca,
+	0x1d704,
+	0x1d73e,
+	0x1d778,
+	0x1d7b2,
+	0x00456,
+	0x0a647,
+	0x004cf,
+	0x0ab75,
+	0x013a5,
+	0x118c3,
+	0x0ff4a,
+	0x02149,
+	0x1d423,
+	0x1d457,
+	0x1d48b,
+	0x1d4bf,
+	0x1d4f3,
+	0x1d527,
+	0x1d55b,
+	0x1d58f,
+	0x1d5c3,
+	0x1d5f7,
+	0x1d62b,
+	0x1d65f,
+	0x1d693,
+	0x003f3,
+	0x00458,
+	0x0ff2a,
+	0x1d409,
+	0x1d43d,
+	0x1d471,
+	0x1d4a5,
+	0x1d4d9,
+	0x1d50d,
+	0x1d541,
+	0x1d575,
+	0x1d5a9,
+	0x1d5dd,
+	0x1d611,
+	0x1d645,
+	0x1d679,
+	0x0a7b2,
+	0x0037f,
+	0x00408,
+	0x013ab,
+	0x0148d,
+	0x0a4d9,
+	0x1d424,
+	0x1d458,
+	0x1d48c,
+	0x1d4c0,
+	0x1d4f4,
+	0x1d528,
+	0x1d55c,
+	0x1d590,
+	0x1d5c4,
+	0x1d5f8,
+	0x1d62c,
+	0x1d660,
+	0x1d694,
+	0x0212a,
+	0x0ff2b,
+	0x1d40a,
+	0x1d43e,
+	0x1d472,
+	0x1d4a6,
+	0x1d4da,
+	0x1d50e,
+	0x1d542,
+	0x1d576,
+	0x1d5aa,
+	0x1d5de,
+	0x1d612,
+	0x1d646,
+	0x1d67a,
+	0x0039a,
+	0x1d6b1,
+	0x1d6eb,
+	0x1d725,
+	0x1d75f,
+	0x1d799,
+	0x02c94,
+	0x0041a,
+	0x013e6,
+	0x016d5,
+	0x0a4d7,
+	0x10518,
+	0x005c0,
+	0x0007c,
+	0x02223,
+	0x023fd,
+	0x0ffe8,
+	0x00031,
+	0x00661,
+	0x006f1,
+	0x10320,
+	0x1e8c7,
+	0x1d7cf,
+	0x1d7d9,
+	0x1d7e3,
+	0x1d7ed,
+	0x1d7f7,
+	0x00049,
+	0x0ff29,
+	0x02160,
+	0x02110,
+	0x02111,
+	0x1d408,
+	0x1d43c,
+	0x1d470,
+	0x1d4d8,
+	0x1d540,
+	0x1d574,
+	0x1d5a8,
+	0x1d5dc,
+	0x1d610,
+	0x1d644,
+	0x1d678,
+	0x00196,
+	0x0ff4c,
+	0x0217c,
+	0x02113,
+	0x1d425,
+	0x1d459,
+	0x1d48d,
+	0x1d4c1,
+	0x1d4f5,
+	0x1d529,
+	0x1d55d,
+	0x1d591,
+	0x1d5c5,
+	0x1d5f9,
+	0x1d62d,
+	0x1d661,
+	0x1d695,
+	0x001c0,
+	0x00399,
+	0x1d6b0,
+	0x1d6ea,
+	0x1d724,
+	0x1d75e,
+	0x1d798,
+	0x02c92,
+	0x00406,
+	0x004c0,
+	0x005d5,
+	0x005df,
+	0x00627,
+	0x1ee00,
+	0x1ee80,
+	0x0fe8e,
+	0x0fe8d,
+	0x007ca,
+	0x02d4f,
+	0x016c1,
+	0x0a4f2,
+	0x16f28,
+	0x1028a,
+	0x10309,
+	0x1d22a,
+	0x0216c,
+	0x02112,
+	0x1d40b,
+	0x1d43f,
+	0x1d473,
+	0x1d4db,
+	0x1d50f,
+	0x1d543,
+	0x1d577,
+	0x1d5ab,
+	0x1d5df,
+	0x1d613,
+	0x1d647,
+	0x1d67b,
+	0x02cd0,
+	0x013de,
+	0x014aa,
+	0x0a4e1,
+	0x16f16,
+	0x118a3,
+	0x118b2,
+	0x1041b,
+	0x10526,
+	0x0ff2d,
+	0x0216f,
+	0x02133,
+	0x1d40c,
+	0x1d440,
+	0x1d474,
+	0x1d4dc,
+	0x1d510,
+	0x1d544,
+	0x1d578,
+	0x1d5ac,
+	0x1d5e0,
+	0x1d614,
+	0x1d648,
+	0x1d67c,
+	0x0039c,
+	0x1d6b3,
+	0x1d6ed,
+	0x1d727,
+	0x1d761,
+	0x1d79b,
+	0x003fa,
+	0x02c98,
+	0x0041c,
+	0x013b7,
+	0x015f0,
+	0x016d6,
+	0x0a4df,
+	0x102b0,
+	0x10311,
+	0x1d427,
+	0x1d45b,
+	0x1d48f,
+	0x1d4c3,
+	0x1d4f7,
+	0x1d52b,
+	0x1d55f,
+	0x1d593,
+	0x1d5c7,
+	0x1d5fb,
+	0x1d62f,
+	0x1d663,
+	0x1d697,
+	0x00578,
+	0x0057c,
+	0x0ff2e,
+	0x02115,
+	0x1d40d,
+	0x1d441,
+	0x1d475,
+	0x1d4a9,
+	0x1d4dd,
+	0x1d511,
+	0x1d579,
+	0x1d5ad,
+	0x1d5e1,
+	0x1d615,
+	0x1d649,
+	0x1d67d,
+	0x0039d,
+	0x1d6b4,
+	0x1d6ee,
+	0x1d728,
+	0x1d762,
+	0x1d79c,
+	0x02c9a,
+	0x0a4e0,
+	0x10513,
+	0x00c02,
+	0x00c82,
+	0x00d02,
+	0x00d82,
+	0x00966,
+	0x00a66,
+	0x00ae6,
+	0x00be6,
+	0x00c66,
+	0x00ce6,
+	0x00d66,
+	0x00e50,
+	0x00ed0,
+	0x01040,
+	0x00665,
+	0x006f5,
+	0x0ff4f,
+	0x02134,
+	0x1d428,
+	0x1d45c,
+	0x1d490,
+	0x1d4f8,
+	0x1d52c,
+	0x1d560,
+	0x1d594,
+	0x1d5c8,
+	0x1d5fc,
+	0x1d630,
+	0x1d664,
+	0x1d698,
+	0x01d0f,
+	0x01d11,
+	0x0ab3d,
+	0x003bf,
+	0x1d6d0,
+	0x1d70a,
+	0x1d744,
+	0x1d77e,
+	0x1d7b8,
+	0x003c3,
+	0x1d6d4,
+	0x1d70e,
+	0x1d748,
+	0x1d782,
+	0x1d7bc,
+	0x02c9f,
+	0x0043e,
+	0x010ff,
+	0x00585,
+	0x005e1,
+	0x00647,
+	0x1ee24,
+	0x1ee64,
+	0x1ee84,
+	0x0feeb,
+	0x0feec,
+	0x0feea,
+	0x0fee9,
+	0x006be,
+	0x0fbac,
+	0x0fbad,
+	0x0fbab,
+	0x0fbaa,
+	0x006c1,
+	0x0fba8,
+	0x0fba9,
+	0x0fba7,
+	0x0fba6,
+	0x006d5,
+	0x00d20,
+	0x0101d,
+	0x104ea,
+	0x118c8,
+	0x118d7,
+	0x1042c,
+	0x00030,
+	0x007c0,
+	0x009e6,
+	0x00b66,
+	0x03007,
+	0x114d0,
+	0x118e0,
+	0x1d7ce,
+	0x1d7d8,
+	0x1d7e2,
+	0x1d7ec,
+	0x1d7f6,
+	0x0ff2f,
+	0x1d40e,
+	0x1d442,
+	0x1d476,
+	0x1d4aa,
+	0x1d4de,
+	0x1d512,
+	0x1d546,
+	0x1d57a,
+	0x1d5ae,
+	0x1d5e2,
+	0x1d616,
+	0x1d64a,
+	0x1d67e,
+	0x0039f,
+	0x1d6b6,
+	0x1d6f0,
+	0x1d72a,
+	0x1d764,
+	0x1d79e,
+	0x02c9e,
+	0x0041e,
+	0x00555,
+	0x02d54,
+	0x012d0,
+	0x00b20,
+	0x104c2,
+	0x0a4f3,
+	0x118b5,
+	0x10292,
+	0x102ab,
+	0x10404,
+	0x10516,
+	0x02374,
+	0x0ff50,
+	0x1d429,
+	0x1d45d,
+	0x1d491,
+	0x1d4c5,
+	0x1d4f9,
+	0x1d52d,
+	0x1d561,
+	0x1d595,
+	0x1d5c9,
+	0x1d5fd,
+	0x1d631,
+	0x1d665,
+	0x1d699,
+	0x003c1,
+	0x003f1,
+	0x1d6d2,
+	0x1d6e0,
+	0x1d70c,
+	0x1d71a,
+	0x1d746,
+	0x1d754,
+	0x1d780,
+	0x1d78e,
+	0x1d7ba,
+	0x1d7c8,
+	0x02ca3,
+	0x00440,
+	0x0ff30,
+	0x02119,
+	0x1d40f,
+	0x1d443,
+	0x1d477,
+	0x1d4ab,
+	0x1d4df,
+	0x1d513,
+	0x1d57b,
+	0x1d5af,
+	0x1d5e3,
+	0x1d617,
+	0x1d64b,
+	0x1d67f,
+	0x003a1,
+	0x1d6b8,
+	0x1d6f2,
+	0x1d72c,
+	0x1d766,
+	0x1d7a0,
+	0x02ca2,
+	0x00420,
+	0x013e2,
+	0x0146d,
+	0x0a4d1,
+	0x10295,
+	0x1d42a,
+	0x1d45e,
+	0x1d492,
+	0x1d4c6,
+	0x1d4fa,
+	0x1d52e,
+	0x1d562,
+	0x1d596,
+	0x1d5ca,
+	0x1d5fe,
+	0x1d632,
+	0x1d666,
+	0x1d69a,
+	0x0051b,
+	0x00563,
+	0x00566,
+	0x0211a,
+	0x1d410,
+	0x1d444,
+	0x1d478,
+	0x1d4ac,
+	0x1d4e0,
+	0x1d514,
+	0x1d57c,
+	0x1d5b0,
+	0x1d5e4,
+	0x1d618,
+	0x1d64c,
+	0x1d680,
+	0x02d55,
+	0x1d42b,
+	0x1d45f,
+	0x1d493,
+	0x1d4c7,
+	0x1d4fb,
+	0x1d52f,
+	0x1d563,
+	0x1d597,
+	0x1d5cb,
+	0x1d5ff,
+	0x1d633,
+	0x1d667,
+	0x1d69b,
+	0x0ab47,
+	0x0ab48,
+	0x01d26,
+	0x02c85,
+	0x00433,
+	0x0ab81,
+	0x1d216,
+	0x0211b,
+	0x0211c,
+	0x0211d,
+	0x1d411,
+	0x1d445,
+	0x1d479,
+	0x1d4e1,
+	0x1d57d,
+	0x1d5b1,
+	0x1d5e5,
+	0x1d619,
+	0x1d64d,
+	0x1d681,
+	0x001a6,
+	0x013a1,
+	0x013d2,
+	0x104b4,
+	0x01587,
+	0x0a4e3,
+	0x16f35,
+	0x0ff53,
+	0x1d42c,
+	0x1d460,
+	0x1d494,
+	0x1d4c8,
+	0x1d4fc,
+	0x1d530,
+	0x1d564,
+	0x1d598,
+	0x1d5cc,
+	0x1d600,
+	0x1d634,
+	0x1d668,
+	0x1d69c,
+	0x0a731,
+	0x001bd,
+	0x00455,
+	0x0abaa,
+	0x118c1,
+	0x10448,
+	0x0ff33,
+	0x1d412,
+	0x1d446,
+	0x1d47a,
+	0x1d4ae,
+	0x1d4e2,
+	0x1d516,
+	0x1d54a,
+	0x1d57e,
+	0x1d5b2,
+	0x1d5e6,
+	0x1d61a,
+	0x1d64e,
+	0x1d682,
+	0x00405,
+	0x0054f,
+	0x013d5,
+	0x013da,
+	0x0a4e2,
+	0x16f3a,
+	0x10296,
+	0x10420,
+	0x1d42d,
+	0x1d461,
+	0x1d495,
+	0x1d4c9,
+	0x1d4fd,
+	0x1d531,
+	0x1d565,
+	0x1d599,
+	0x1d5cd,
+	0x1d601,
+	0x1d635,
+	0x1d669,
+	0x1d69d,
+	0x022a4,
+	0x027d9,
+	0x1f768,
+	0x0ff34,
+	0x1d413,
+	0x1d447,
+	0x1d47b,
+	0x1d4af,
+	0x1d4e3,
+	0x1d517,
+	0x1d54b,
+	0x1d57f,
+	0x1d5b3,
+	0x1d5e7,
+	0x1d61b,
+	0x1d64f,
+	0x1d683,
+	0x003a4,
+	0x1d6bb,
+	0x1d6f5,
+	0x1d72f,
+	0x1d769,
+	0x1d7a3,
+	0x02ca6,
+	0x00422,
+	0x013a2,
+	0x0a4d4,
+	0x16f0a,
+	0x118bc,
+	0x10297,
+	0x102b1,
+	0x10315,
+	0x1d42e,
+	0x1d462,
+	0x1d496,
+	0x1d4ca,
+	0x1d4fe,
+	0x1d532,
+	0x1d566,
+	0x1d59a,
+	0x1d5ce,
+	0x1d602,
+	0x1d636,
+	0x1d66a,
+	0x1d69e,
+	0x0a79f,
+	0x01d1c,
+	0x0ab4e,
+	0x0ab52,
+	0x0028b,
+	0x003c5,
+	0x1d6d6,
+	0x1d710,
+	0x1d74a,
+	0x1d784,
+	0x1d7be,
+	0x0057d,
+	0x104f6,
+	0x118d8,
+	0x0222a,
+	0x022c3,
+	0x1d414,
+	0x1d448,
+	0x1d47c,
+	0x1d4b0,
+	0x1d4e4,
+	0x1d518,
+	0x1d54c,
+	0x1d580,
+	0x1d5b4,
+	0x1d5e8,
+	0x1d61c,
+	0x1d650,
+	0x1d684,
+	0x0054d,
+	0x01200,
+	0x104ce,
+	0x0144c,
+	0x0a4f4,
+	0x16f42,
+	0x118b8,
+	0x02228,
+	0x022c1,
+	0x0ff56,
+	0x02174,
+	0x1d42f,
+	0x1d463,
+	0x1d497,
+	0x1d4cb,
+	0x1d4ff,
+	0x1d533,
+	0x1d567,
+	0x1d59b,
+	0x1d5cf,
+	0x1d603,
+	0x1d637,
+	0x1d66b,
+	0x1d69f,
+	0x01d20,
+	0x003bd,
+	0x1d6ce,
+	0x1d708,
+	0x1d742,
+	0x1d77c,
+	0x1d7b6,
+	0x00475,
+	0x005d8,
+	0x11706,
+	0x0aba9,
+	0x118c0,
+	0x1d20d,
+	0x00667,
+	0x006f7,
+	0x02164,
+	0x1d415,
+	0x1d449,
+	0x1d47d,
+	0x1d4b1,
+	0x1d4e5,
+	0x1d519,
+	0x1d54d,
+	0x1d581,
+	0x1d5b5,
+	0x1d5e9,
+	0x1d61d,
+	0x1d651,
+	0x1d685,
+	0x00474,
+	0x02d38,
+	0x013d9,
+	0x0142f,
+	0x0a6df,
+	0x0a4e6,
+	0x16f08,
+	0x118a0,
+	0x1051d,
+	0x0026f,
+	0x1d430,
+	0x1d464,
+	0x1d498,
+	0x1d4cc,
+	0x1d500,
+	0x1d534,
+	0x1d568,
+	0x1d59c,
+	0x1d5d0,
+	0x1d604,
+	0x1d638,
+	0x1d66c,
+	0x1d6a0,
+	0x01d21,
+	0x00461,
+	0x0051d,
+	0x00561,
+	0x1170a,
+	0x1170e,
+	0x1170f,
+	0x0ab83,
+	0x118ef,
+	0x118e6,
+	0x1d416,
+	0x1d44a,
+	0x1d47e,
+	0x1d4b2,
+	0x1d4e6,
+	0x1d51a,
+	0x1d54e,
+	0x1d582,
+	0x1d5b6,
+	0x1d5ea,
+	0x1d61e,
+	0x1d652,
+	0x1d686,
+	0x0051c,
+	0x013b3,
+	0x013d4,
+	0x0a4ea,
+	0x0166e,
+	0x000d7,
+	0x0292b,
+	0x0292c,
+	0x02a2f,
+	0x0ff58,
+	0x02179,
+	0x1d431,
+	0x1d465,
+	0x1d499,
+	0x1d4cd,
+	0x1d501,
+	0x1d535,
+	0x1d569,
+	0x1d59d,
+	0x1d5d1,
+	0x1d605,
+	0x1d639,
+	0x1d66d,
+	0x1d6a1,
+	0x00445,
+	0x01541,
+	0x0157d,
+	0x0166d,
+	0x02573,
+	0x10322,
+	0x118ec,
+	0x0ff38,
+	0x02169,
+	0x1d417,
+	0x1d44b,
+	0x1d47f,
+	0x1d4b3,
+	0x1d4e7,
+	0x1d51b,
+	0x1d54f,
+	0x1d583,
+	0x1d5b7,
+	0x1d5eb,
+	0x1d61f,
+	0x1d653,
+	0x1d687,
+	0x0a7b3,
+	0x003a7,
+	0x1d6be,
+	0x1d6f8,
+	0x1d732,
+	0x1d76c,
+	0x1d7a6,
+	0x02cac,
+	0x00425,
+	0x02d5d,
+	0x016b7,
+	0x0a4eb,
+	0x10290,
+	0x102b4,
+	0x10317,
+	0x10527,
+	0x00263,
+	0x01d8c,
+	0x0ff59,
+	0x1d432,
+	0x1d466,
+	0x1d49a,
+	0x1d4ce,
+	0x1d502,
+	0x1d536,
+	0x1d56a,
+	0x1d59e,
+	0x1d5d2,
+	0x1d606,
+	0x1d63a,
+	0x1d66e,
+	0x1d6a2,
+	0x0028f,
+	0x01eff,
+	0x0ab5a,
+	0x003b3,
+	0x0213d,
+	0x1d6c4,
+	0x1d6fe,
+	0x1d738,
+	0x1d772,
+	0x1d7ac,
+	0x00443,
+	0x004af,
+	0x010e7,
+	0x118dc,
+	0x0ff39,
+	0x1d418,
+	0x1d44c,
+	0x1d480,
+	0x1d4b4,
+	0x1d4e8,
+	0x1d51c,
+	0x1d550,
+	0x1d584,
+	0x1d5b8,
+	0x1d5ec,
+	0x1d620,
+	0x1d654,
+	0x1d688,
+	0x003a5,
+	0x003d2,
+	0x1d6bc,
+	0x1d6f6,
+	0x1d730,
+	0x1d76a,
+	0x1d7a4,
+	0x02ca8,
+	0x00423,
+	0x004ae,
+	0x013a9,
+	0x013bd,
+	0x0a4ec,
+	0x16f43,
+	0x118a4,
+	0x102b2,
+	0x1d433,
+	0x1d467,
+	0x1d49b,
+	0x1d4cf,
+	0x1d503,
+	0x1d537,
+	0x1d56b,
+	0x1d59f,
+	0x1d5d3,
+	0x1d607,
+	0x1d63b,
+	0x1d66f,
+	0x1d6a3,
+	0x01d22,
+	0x0ab93,
+	0x118c4,
+	0x102f5,
+	0x118e5,
+	0x0ff3a,
+	0x02124,
+	0x02128,
+	0x1d419,
+	0x1d44d,
+	0x1d481,
+	0x1d4b5,
+	0x1d4e9,
+	0x1d585,
+	0x1d5b9,
+	0x1d5ed,
+	0x1d621,
+	0x1d655,
+	0x1d689,
+	0x00396,
+	0x1d6ad,
+	0x1d6e7,
+	0x1d721,
+	0x1d75b,
+	0x1d795,
+	0x013c3,
+	0x0a4dc,
+	0x118a9,
+};
+
+static gboolean
+rspamd_can_alias_latin(gint ch)
+{
+	return latin_confusable.contains(ch);
+}
+
+static gdouble
+rspamd_chartable_process_word_utf(struct rspamd_task *task,
+								  rspamd_stat_token_t *w,
+								  gboolean is_url,
+								  guint *ncap,
+								  struct chartable_ctx *chartable_module_ctx,
+								  gboolean ignore_diacritics)
+{
+	const UChar32 *p, *end;
+	gdouble badness = 0.0;
+	UChar32 uc;
+	UBlockCode sc;
+	guint cat;
+	gint last_is_latin = -1;
+	guint same_script_count = 0, nsym = 0, nspecial = 0;
+	enum {
+		start_process = 0,
+		got_alpha,
+		got_digit,
+		got_unknown,
+	} state = start_process,
+	  prev_state = start_process;
+
+	p = w->unicode.begin;
+	end = p + w->unicode.len;
+
+	/* We assume that w is normalized */
+
+	while (p < end) {
+		uc = *p++;
+
+		if (((gint32) uc) < 0) {
+			break;
+		}
+
+		sc = ublock_getCode(uc);
+		cat = u_charType(uc);
+
+		if (!ignore_diacritics) {
+			if (cat == U_NON_SPACING_MARK ||
+				(sc == UBLOCK_LATIN_1_SUPPLEMENT) ||
+				(sc == UBLOCK_LATIN_EXTENDED_A) ||
+				(sc == UBLOCK_LATIN_EXTENDED_ADDITIONAL) ||
+				(sc == UBLOCK_LATIN_EXTENDED_B) ||
+				(sc == UBLOCK_COMBINING_DIACRITICAL_MARKS)) {
+				nspecial++;
+			}
+		}
+
+		if (u_isalpha(uc)) {
+
+			if (sc <= UBLOCK_COMBINING_DIACRITICAL_MARKS ||
+				sc == UBLOCK_LATIN_EXTENDED_ADDITIONAL) {
+				/*
+				 * Assume all latin, IPA, diacritic and space modifiers
+				 * characters as basic latin
+				 */
+				sc = UBLOCK_BASIC_LATIN;
+			}
+
+			if (sc != UBLOCK_BASIC_LATIN && u_isupper(uc)) {
+				if (ncap) {
+					(*ncap)++;
+				}
+			}
+
+			if (state == got_digit) {
+				/* Penalize digit -> alpha translations */
+				if (!is_url && sc != UBLOCK_BASIC_LATIN &&
+					prev_state != start_process) {
+					badness += 0.25;
+				}
+			}
+			else if (state == got_alpha) {
+				/* Check script */
+				if (same_script_count > 0) {
+					if (sc != UBLOCK_BASIC_LATIN && last_is_latin) {
+
+						if (rspamd_can_alias_latin(uc)) {
+							badness += 1.0 / (gdouble) same_script_count;
+						}
+
+						last_is_latin = 0;
+						same_script_count = 1;
+					}
+					else {
+						same_script_count++;
+					}
+				}
+				else {
+					last_is_latin = sc == UBLOCK_BASIC_LATIN;
+					same_script_count = 1;
+				}
+			}
+
+			prev_state = state;
+			state = got_alpha;
+		}
+		else if (u_isdigit(uc)) {
+			if (state != got_digit) {
+				prev_state = state;
+			}
+
+			state = got_digit;
+			same_script_count = 0;
+		}
+		else {
+			/* We don't care about unknown characters here */
+			if (state != got_unknown) {
+				prev_state = state;
+			}
+
+			state = got_unknown;
+			same_script_count = 0;
+		}
+
+		nsym++;
+	}
+
+	if (nspecial > 0) {
+		if (!ignore_diacritics) {
+			/* Count diacritics  */
+			badness += nspecial;
+		}
+		else if (nspecial > 1) {
+			badness += (nspecial - 1.0) / 2.0;
+		}
+	}
+
+	/* Try to avoid FP for long words */
+	if (nsym > chartable_module_ctx->max_word_len) {
+		badness = 0;
+	}
+	else {
+		if (badness > 4.0) {
+			badness = 4.0;
+		}
+	}
+
+	msg_debug_chartable("word %*s, badness: %.2f",
+						(gint) w->normalized.len, w->normalized.begin,
+						badness);
+
+	return badness;
+}
+
+static gdouble
+rspamd_chartable_process_word_ascii(struct rspamd_task *task,
+									rspamd_stat_token_t *w,
+									gboolean is_url,
+									struct chartable_ctx *chartable_module_ctx)
+{
+	gdouble badness = 0.0;
+	enum {
+		ascii = 1,
+		non_ascii
+	} sc,
+		last_sc;
+	gint same_script_count = 0, seen_alpha = FALSE;
+	enum {
+		start_process = 0,
+		got_alpha,
+		got_digit,
+		got_unknown,
+	} state = start_process;
+
+	const auto *p = (const unsigned char *) w->normalized.begin;
+	const auto *end = p + w->normalized.len;
+	last_sc = non_ascii;
+
+	if (w->normalized.len > chartable_module_ctx->max_word_len) {
+		return 0.0;
+	}
+
+	/* We assume that w is normalized */
+	while (p < end) {
+		if (g_ascii_isalpha(*p) || *p > 0x7f) {
+
+			if (state == got_digit) {
+				/* Penalize digit -> alpha translations */
+				if (seen_alpha && !is_url && !g_ascii_isxdigit(*p)) {
+					badness += 0.25;
+				}
+			}
+			else if (state == got_alpha) {
+				/* Check script */
+				sc = (*p > 0x7f) ? ascii : non_ascii;
+
+				if (same_script_count > 0) {
+					if (sc != last_sc) {
+						badness += 1.0 / (gdouble) same_script_count;
+						last_sc = sc;
+						same_script_count = 1;
+					}
+					else {
+						same_script_count++;
+					}
+				}
+				else {
+					last_sc = sc;
+					same_script_count = 1;
+				}
+			}
+
+			seen_alpha = TRUE;
+			state = got_alpha;
+		}
+		else if (g_ascii_isdigit(*p)) {
+			state = got_digit;
+			same_script_count = 0;
+		}
+		else {
+			/* We don't care about unknown characters here */
+			state = got_unknown;
+			same_script_count = 0;
+		}
+
+		p++;
+	}
+
+	if (badness > 4.0) {
+		badness = 4.0;
+	}
+
+	msg_debug_chartable("word %*s, badness: %.2f",
+						(gint) w->normalized.len, w->normalized.begin,
+						badness);
+
+	return badness;
+}
+
+static gboolean
+rspamd_chartable_process_part(struct rspamd_task *task,
+							  struct rspamd_mime_text_part *part,
+							  struct chartable_ctx *chartable_module_ctx,
+							  gboolean ignore_diacritics)
+{
+	rspamd_stat_token_t *w;
+	guint i, ncap = 0;
+	gdouble cur_score = 0.0;
+
+	if (part == nullptr || part->utf_words == nullptr ||
+		part->utf_words->len == 0 || part->nwords == 0) {
+		return FALSE;
+	}
+
+	for (i = 0; i < part->utf_words->len; i++) {
+		w = &g_array_index(part->utf_words, rspamd_stat_token_t, i);
+
+		if ((w->flags & RSPAMD_STAT_TOKEN_FLAG_TEXT)) {
+
+			if (w->flags & RSPAMD_STAT_TOKEN_FLAG_UTF) {
+				cur_score += rspamd_chartable_process_word_utf(task, w, FALSE,
+															   &ncap, chartable_module_ctx, ignore_diacritics);
+			}
+			else {
+				cur_score += rspamd_chartable_process_word_ascii(task, w,
+																 FALSE, chartable_module_ctx);
+			}
+		}
+	}
+
+	/*
+	 * TODO: perhaps, we should do this analysis somewhere else and get
+	 * something like: <SYM_SC><SYM_SC><SYM_SC> representing classes for all
+	 * symbols in the text
+	 */
+	part->capital_letters += ncap;
+
+	cur_score /= (gdouble) part->nwords;
+
+	if (cur_score > 1.0) {
+		cur_score = 1.0;
+	}
+
+	if (cur_score > chartable_module_ctx->threshold) {
+		rspamd_task_insert_result(task, chartable_module_ctx->symbol,
+								  cur_score, nullptr);
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static void
+chartable_symbol_callback(struct rspamd_task *task,
+						  struct rspamd_symcache_dynamic_item *item,
+						  void *_)
+{
+	guint i;
+	struct rspamd_mime_text_part *part;
+	struct chartable_ctx *chartable_module_ctx = chartable_get_context(task->cfg);
+	gboolean ignore_diacritics = TRUE, seen_violated_part = FALSE;
+
+	/* Check if we have parts with diacritic symbols language */
+	PTR_ARRAY_FOREACH(MESSAGE_FIELD(task, text_parts), i, part)
+	{
+		if (part->languages && part->languages->len > 0) {
+			auto *lang = (struct rspamd_lang_detector_res *) g_ptr_array_index(part->languages, 0);
+			gint flags;
+
+			flags = rspamd_language_detector_elt_flags(lang->elt);
+
+			if ((flags & RS_LANGUAGE_DIACRITICS)) {
+				ignore_diacritics = TRUE;
+			}
+			else if (lang->prob > 0.75) {
+				ignore_diacritics = FALSE;
+			}
+		}
+
+		if (rspamd_chartable_process_part(task, part, chartable_module_ctx, ignore_diacritics)) {
+			seen_violated_part = TRUE;
+		}
+	}
+
+	if (MESSAGE_FIELD(task, text_parts)->len == 0) {
+		/* No text parts, assume that we should ignore diacritics checks for metatokens */
+		ignore_diacritics = TRUE;
+	}
+
+	if (task->meta_words != nullptr && task->meta_words->len > 0) {
+		rspamd_stat_token_t *w;
+		gdouble cur_score = 0;
+		gsize arlen = task->meta_words->len;
+
+		for (i = 0; i < arlen; i++) {
+			w = &g_array_index(task->meta_words, rspamd_stat_token_t, i);
+			cur_score += rspamd_chartable_process_word_utf(task, w, FALSE,
+														   nullptr, chartable_module_ctx, ignore_diacritics);
+		}
+
+		cur_score /= (gdouble) (arlen + 1);
+
+		if (cur_score > 1.0) {
+			cur_score = 1.0;
+		}
+
+		if (cur_score > chartable_module_ctx->threshold) {
+			if (!seen_violated_part) {
+				/* Further penalise */
+				if (cur_score > 0.25) {
+					cur_score = 0.25;
+				}
+			}
+
+			rspamd_task_insert_result(task, chartable_module_ctx->symbol,
+									  cur_score, "subject");
+		}
+	}
+
+	rspamd_symcache_finalize_item(task, item);
+}
+
+static void
+chartable_url_symbol_callback(struct rspamd_task *task,
+							  struct rspamd_symcache_dynamic_item *item,
+							  void *unused)
+{
+	/* XXX: TODO: unbreak module once URLs unicode project is over */
+#if 0
+	struct rspamd_url *u;
+	GHashTableIter it;
+	gpointer k, v;
+	rspamd_stat_token_t w;
+	gdouble cur_score = 0.0;
+	struct chartable_ctx *chartable_module_ctx = chartable_get_context (task->cfg);
+
+	g_hash_table_iter_init (&it, task->urls);
+
+	while (g_hash_table_iter_next (&it, &k, &v)) {
+		u = v;
+
+		if (cur_score > 2.0) {
+			cur_score = 2.0;
+			break;
+		}
+
+		if (u->hostlen > 0) {
+			w.stemmed.begin = u->host;
+			w.stemmed.len = u->hostlen;
+
+			if (g_utf8_validate (w.stemmed.begin, w.stemmed.len, nullptr)) {
+				cur_score += rspamd_chartable_process_word_utf (task, &w,
+						TRUE, nullptr, chartable_module_ctx);
+			}
+			else {
+				cur_score += rspamd_chartable_process_word_ascii (task, &w,
+						TRUE, chartable_module_ctx);
+			}
+		}
+	}
+
+	g_hash_table_iter_init (&it, task->emails);
+
+	while (g_hash_table_iter_next (&it, &k, &v)) {
+		u = v;
+
+		if (cur_score > 2.0) {
+			cur_score = 2.0;
+			break;
+		}
+
+		if (u->hostlen > 0) {
+			w.stemmed.begin = u->host;
+			w.stemmed.len = u->hostlen;
+
+			if (g_utf8_validate (w.stemmed.begin, w.stemmed.len, nullptr)) {
+				cur_score += rspamd_chartable_process_word_utf (task, &w,
+						TRUE, nullptr, chartable_module_ctx);
+			}
+			else {
+				cur_score += rspamd_chartable_process_word_ascii (task, &w,
+						TRUE, chartable_module_ctx);
+			}
+		}
+	}
+
+	if (cur_score > chartable_module_ctx->threshold) {
+		rspamd_task_insert_result (task, chartable_module_ctx->symbol,
+				cur_score, nullptr);
+
+	}
+#endif
+	rspamd_symcache_finalize_item(task, item);
+}