Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 4365 insertions, 0 deletions

diff --git a/src/libserver/url.c b/src/libserver/url.c
new file mode 100644
index 0000000..0842a1e
--- /dev/null
+++ b/src/libserver/url.c
@@ -0,0 +1,4365 @@
+/*
+ * Copyright 2023 Vsevolod Stakhov
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "config.h"
+#include "url.h"
+#include "util.h"
+#include "rspamd.h"
+#include "message.h"
+#include "multipattern.h"
+#include "contrib/uthash/utlist.h"
+#include "contrib/http-parser/http_parser.h"
+#include <unicode/utf8.h>
+#include <unicode/uchar.h>
+#include <unicode/usprep.h>
+#include <unicode/ucnv.h>
+
+typedef struct url_match_s {
+	const gchar *m_begin;
+	gsize m_len;
+	const gchar *pattern;
+	const gchar *prefix;
+	const gchar *newline_pos;
+	const gchar *prev_newline_pos;
+	gboolean add_prefix;
+	gchar st;
+} url_match_t;
+
+#define URL_MATCHER_FLAG_NOHTML (1u << 0u)
+#define URL_MATCHER_FLAG_TLD_MATCH (1u << 1u)
+#define URL_MATCHER_FLAG_STAR_MATCH (1u << 2u)
+#define URL_MATCHER_FLAG_REGEXP (1u << 3u)
+
+struct url_callback_data;
+
+static const struct {
+	enum rspamd_url_protocol proto;
+	const gchar *name;
+	gsize len;
+} rspamd_url_protocols[] = {
+	{.proto = PROTOCOL_FILE,
+	 .name = "file",
+	 .len = 4},
+	{.proto = PROTOCOL_FTP,
+	 .name = "ftp",
+	 .len = 3},
+	{.proto = PROTOCOL_HTTP,
+	 .name = "http",
+	 .len = 4},
+	{.proto = PROTOCOL_HTTPS,
+	 .name = "https",
+	 .len = 5},
+	{.proto = PROTOCOL_MAILTO,
+	 .name = "mailto",
+	 .len = 6},
+	{.proto = PROTOCOL_TELEPHONE,
+	 .name = "tel",
+	 .len = 3},
+	{.proto = PROTOCOL_TELEPHONE,
+	 .name = "callto",
+	 .len = 3},
+	{.proto = PROTOCOL_UNKNOWN,
+	 .name = NULL,
+	 .len = 0}};
+struct url_matcher {
+	const gchar *pattern;
+	const gchar *prefix;
+
+	gboolean (*start)(struct url_callback_data *cb,
+					  const gchar *pos,
+					  url_match_t *match);
+
+	gboolean (*end)(struct url_callback_data *cb,
+					const gchar *pos,
+					url_match_t *match);
+
+	gint flags;
+};
+
+static gboolean url_file_start(struct url_callback_data *cb,
+							   const gchar *pos,
+							   url_match_t *match);
+
+static gboolean url_file_end(struct url_callback_data *cb,
+							 const gchar *pos,
+							 url_match_t *match);
+
+static gboolean url_web_start(struct url_callback_data *cb,
+							  const gchar *pos,
+							  url_match_t *match);
+
+static gboolean url_web_end(struct url_callback_data *cb,
+							const gchar *pos,
+							url_match_t *match);
+
+static gboolean url_tld_start(struct url_callback_data *cb,
+							  const gchar *pos,
+							  url_match_t *match);
+
+static gboolean url_tld_end(struct url_callback_data *cb,
+							const gchar *pos,
+							url_match_t *match);
+
+static gboolean url_email_start(struct url_callback_data *cb,
+								const gchar *pos,
+								url_match_t *match);
+
+static gboolean url_email_end(struct url_callback_data *cb,
+							  const gchar *pos,
+							  url_match_t *match);
+
+static gboolean url_tel_start(struct url_callback_data *cb,
+							  const gchar *pos,
+							  url_match_t *match);
+
+static gboolean url_tel_end(struct url_callback_data *cb,
+							const gchar *pos,
+							url_match_t *match);
+
+struct url_matcher static_matchers[] = {
+	/* Common prefixes */
+	{"file://", "", url_file_start, url_file_end,
+	 0},
+	{"file:\\\\", "", url_file_start, url_file_end,
+	 0},
+	{"ftp://", "", url_web_start, url_web_end,
+	 0},
+	{"ftp:\\\\", "", url_web_start, url_web_end,
+	 0},
+	{"sftp://", "", url_web_start, url_web_end,
+	 0},
+	{"http:", "", url_web_start, url_web_end,
+	 0},
+	{"https:", "", url_web_start, url_web_end,
+	 0},
+	{"news://", "", url_web_start, url_web_end,
+	 0},
+	{"nntp://", "", url_web_start, url_web_end,
+	 0},
+	{"telnet://", "", url_web_start, url_web_end,
+	 0},
+	{"tel:", "", url_tel_start, url_tel_end,
+	 0},
+	{"webcal://", "", url_web_start, url_web_end,
+	 0},
+	{"mailto:", "", url_email_start, url_email_end,
+	 0},
+	{"callto:", "", url_tel_start, url_tel_end,
+	 0},
+	{"h323:", "", url_web_start, url_web_end,
+	 0},
+	{"sip:", "", url_web_start, url_web_end,
+	 0},
+	{"www\\.[0-9a-z]", "http://", url_web_start, url_web_end,
+	 URL_MATCHER_FLAG_REGEXP},
+	{"ftp.", "ftp://", url_web_start, url_web_end,
+	 0},
+	/* Likely emails */
+	{
+		"@", "mailto://", url_email_start, url_email_end,
+		0}};
+
+struct rspamd_url_flag_name {
+	const gchar *name;
+	gint flag;
+	gint hash;
+} url_flag_names[] = {
+	{"phished", RSPAMD_URL_FLAG_PHISHED, -1},
+	{"numeric", RSPAMD_URL_FLAG_NUMERIC, -1},
+	{"obscured", RSPAMD_URL_FLAG_OBSCURED, -1},
+	{"redirected", RSPAMD_URL_FLAG_REDIRECTED, -1},
+	{"html_displayed", RSPAMD_URL_FLAG_HTML_DISPLAYED, -1},
+	{"text", RSPAMD_URL_FLAG_FROM_TEXT, -1},
+	{"subject", RSPAMD_URL_FLAG_SUBJECT, -1},
+	{"host_encoded", RSPAMD_URL_FLAG_HOSTENCODED, -1},
+	{"schema_encoded", RSPAMD_URL_FLAG_SCHEMAENCODED, -1},
+	{"path_encoded", RSPAMD_URL_FLAG_PATHENCODED, -1},
+	{"query_encoded", RSPAMD_URL_FLAG_QUERYENCODED, -1},
+	{"missing_slashes", RSPAMD_URL_FLAG_MISSINGSLASHES, -1},
+	{"idn", RSPAMD_URL_FLAG_IDN, -1},
+	{"has_port", RSPAMD_URL_FLAG_HAS_PORT, -1},
+	{"has_user", RSPAMD_URL_FLAG_HAS_USER, -1},
+	{"schemaless", RSPAMD_URL_FLAG_SCHEMALESS, -1},
+	{"unnormalised", RSPAMD_URL_FLAG_UNNORMALISED, -1},
+	{"zw_spaces", RSPAMD_URL_FLAG_ZW_SPACES, -1},
+	{"url_displayed", RSPAMD_URL_FLAG_DISPLAY_URL, -1},
+	{"image", RSPAMD_URL_FLAG_IMAGE, -1},
+	{"query", RSPAMD_URL_FLAG_QUERY, -1},
+	{"content", RSPAMD_URL_FLAG_CONTENT, -1},
+	{"no_tld", RSPAMD_URL_FLAG_NO_TLD, -1},
+	{"truncated", RSPAMD_URL_FLAG_TRUNCATED, -1},
+	{"redirect_target", RSPAMD_URL_FLAG_REDIRECT_TARGET, -1},
+	{"invisible", RSPAMD_URL_FLAG_INVISIBLE, -1},
+	{"special", RSPAMD_URL_FLAG_SPECIAL, -1},
+};
+
+
+static inline khint_t rspamd_url_hash(struct rspamd_url *u);
+
+static inline khint_t rspamd_url_host_hash(struct rspamd_url *u);
+static inline bool rspamd_urls_cmp(struct rspamd_url *a, struct rspamd_url *b);
+static inline bool rspamd_urls_host_cmp(struct rspamd_url *a, struct rspamd_url *b);
+
+/* Hash table implementation */
+__KHASH_IMPL(rspamd_url_hash, kh_inline, struct rspamd_url *, char, false,
+			 rspamd_url_hash, rspamd_urls_cmp);
+__KHASH_IMPL(rspamd_url_host_hash, kh_inline, struct rspamd_url *, char, false,
+			 rspamd_url_host_hash, rspamd_urls_host_cmp);
+
+struct url_callback_data {
+	const gchar *begin;
+	gchar *url_str;
+	rspamd_mempool_t *pool;
+	gint len;
+	enum rspamd_url_find_type how;
+	gboolean prefix_added;
+	guint newline_idx;
+	GArray *matchers;
+	GPtrArray *newlines;
+	const gchar *start;
+	const gchar *fin;
+	const gchar *end;
+	const gchar *last_at;
+	url_insert_function func;
+	void *funcd;
+};
+
+struct url_match_scanner {
+	GArray *matchers_full;
+	GArray *matchers_strict;
+	struct rspamd_multipattern *search_trie_full;
+	struct rspamd_multipattern *search_trie_strict;
+	bool has_tld_file;
+};
+
+struct url_match_scanner *url_scanner = NULL;
+
+enum {
+	IS_LWSP = (1 << 0),
+	IS_DOMAIN = (1 << 1),
+	IS_URLSAFE = (1 << 2),
+	IS_MAILSAFE = (1 << 3),
+	IS_DOMAIN_END = (1 << 4)
+};
+
+static const unsigned int url_scanner_table[256] = {
+	0, 0, 0, 0, 0, 0, 0, 0, 0, IS_LWSP, IS_LWSP, IS_LWSP, IS_LWSP, IS_LWSP, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, IS_LWSP /*   */,
+	IS_MAILSAFE /* ! */, IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* " */,
+	IS_MAILSAFE /* # */, IS_MAILSAFE /* $ */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* % */, 0 /* & */, IS_MAILSAFE /* ' */,
+	0 /* ( */, 0 /* ) */, IS_MAILSAFE /* * */,
+	IS_MAILSAFE /* + */, IS_MAILSAFE /* , */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* - */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* . */, IS_DOMAIN_END | IS_MAILSAFE /* / */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 0 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 1 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 2 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 3 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 4 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 5 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 6 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 7 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 8 */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* 9 */, IS_DOMAIN_END /* : */,
+	0 /* ; */, IS_URLSAFE | IS_DOMAIN_END /* < */, 0 /* = */,
+	IS_URLSAFE | IS_DOMAIN_END /* > */, IS_DOMAIN_END /* ? */, 0 /* @ */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* A */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* B */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* C */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* D */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* E */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* F */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* G */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* H */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* I */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* J */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* K */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* L */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* M */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* N */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* O */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* P */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* Q */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* R */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* S */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* T */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* U */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* V */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* W */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* X */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* Y */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* Z */, 0 /* [ */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* \ */, 0 /* ] */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* ^ */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* _ */,
+	IS_URLSAFE | IS_DOMAIN_END /* ` */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* a */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* b */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* c */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* d */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* e */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* f */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* g */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* h */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* i */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* j */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* k */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* l */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* m */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* n */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* o */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* p */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* q */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* r */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* s */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* t */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* u */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* v */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* w */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* x */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* y */,
+	IS_URLSAFE | IS_DOMAIN | IS_MAILSAFE /* z */,
+	IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* { */,
+	IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* | */,
+	IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* } */,
+	IS_URLSAFE | IS_DOMAIN_END | IS_MAILSAFE /* ~ */, 0, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN, IS_URLSAFE | IS_DOMAIN,
+	IS_URLSAFE | IS_DOMAIN};
+
+#define is_lwsp(x) ((url_scanner_table[(guchar) (x)] & IS_LWSP) != 0)
+#define is_mailsafe(x) ((url_scanner_table[(guchar) (x)] & (IS_MAILSAFE)) != 0)
+#define is_domain(x) ((url_scanner_table[(guchar) (x)] & IS_DOMAIN) != 0)
+#define is_urlsafe(x) ((url_scanner_table[(guchar) (x)] & (IS_URLSAFE)) != 0)
+
+const gchar *
+rspamd_url_strerror(int err)
+{
+	switch (err) {
+	case URI_ERRNO_OK:
+		return "Parsing went well";
+	case URI_ERRNO_EMPTY:
+		return "The URI string was empty";
+	case URI_ERRNO_INVALID_PROTOCOL:
+		return "No protocol was found";
+	case URI_ERRNO_BAD_FORMAT:
+		return "Bad URL format";
+	case URI_ERRNO_BAD_ENCODING:
+		return "Invalid symbols encoded";
+	case URI_ERRNO_INVALID_PORT:
+		return "Port number is bad";
+	case URI_ERRNO_TLD_MISSING:
+		return "TLD part is not detected";
+	case URI_ERRNO_HOST_MISSING:
+		return "Host part is missing";
+	case URI_ERRNO_TOO_LONG:
+		return "URL is too long";
+	}
+
+	return NULL;
+}
+
+static gboolean
+rspamd_url_parse_tld_file(const gchar *fname,
+						  struct url_match_scanner *scanner)
+{
+	FILE *f;
+	struct url_matcher m;
+	gchar *linebuf = NULL, *p;
+	gsize buflen = 0;
+	gssize r;
+	gint flags;
+
+	f = fopen(fname, "r");
+
+	if (f == NULL) {
+		msg_err("cannot open TLD file %s: %s", fname, strerror(errno));
+		return FALSE;
+	}
+
+	m.end = url_tld_end;
+	m.start = url_tld_start;
+	m.prefix = "http://";
+
+	while ((r = getline(&linebuf, &buflen, f)) > 0) {
+		if (linebuf[0] == '/' || g_ascii_isspace(linebuf[0])) {
+			/* Skip comment or empty line */
+			continue;
+		}
+
+		g_strchomp(linebuf);
+
+		/* TODO: add support for ! patterns */
+		if (linebuf[0] == '!') {
+			msg_debug("skip '!' patterns from parsing for now: %s", linebuf);
+			continue;
+		}
+
+		flags = URL_MATCHER_FLAG_NOHTML | URL_MATCHER_FLAG_TLD_MATCH;
+
+		if (linebuf[0] == '*') {
+			flags |= URL_MATCHER_FLAG_STAR_MATCH;
+			p = strchr(linebuf, '.');
+
+			if (p == NULL) {
+				msg_err("got bad star line, skip it: %s", linebuf);
+				continue;
+			}
+			p++;
+		}
+		else {
+			p = linebuf;
+		}
+
+		m.flags = flags;
+		rspamd_multipattern_add_pattern(url_scanner->search_trie_full, p,
+										RSPAMD_MULTIPATTERN_TLD | RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+		m.pattern = rspamd_multipattern_get_pattern(url_scanner->search_trie_full,
+													rspamd_multipattern_get_npatterns(url_scanner->search_trie_full) - 1);
+
+		g_array_append_val(url_scanner->matchers_full, m);
+	}
+
+	free(linebuf);
+	fclose(f);
+
+	return TRUE;
+}
+
+static void
+rspamd_url_add_static_matchers(struct url_match_scanner *sc)
+{
+	gint n = G_N_ELEMENTS(static_matchers), i;
+
+	for (i = 0; i < n; i++) {
+		if (static_matchers[i].flags & URL_MATCHER_FLAG_REGEXP) {
+			rspamd_multipattern_add_pattern(url_scanner->search_trie_strict,
+											static_matchers[i].pattern,
+											RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8 |
+												RSPAMD_MULTIPATTERN_RE);
+		}
+		else {
+			rspamd_multipattern_add_pattern(url_scanner->search_trie_strict,
+											static_matchers[i].pattern,
+											RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+		}
+	}
+
+	g_array_append_vals(sc->matchers_strict, static_matchers, n);
+
+	if (sc->matchers_full) {
+		for (i = 0; i < n; i++) {
+			if (static_matchers[i].flags & URL_MATCHER_FLAG_REGEXP) {
+				rspamd_multipattern_add_pattern(url_scanner->search_trie_full,
+												static_matchers[i].pattern,
+												RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8 |
+													RSPAMD_MULTIPATTERN_RE);
+			}
+			else {
+				rspamd_multipattern_add_pattern(url_scanner->search_trie_full,
+												static_matchers[i].pattern,
+												RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+			}
+		}
+		g_array_append_vals(sc->matchers_full, static_matchers, n);
+	}
+}
+
+void rspamd_url_deinit(void)
+{
+	if (url_scanner != NULL) {
+		if (url_scanner->search_trie_full) {
+			rspamd_multipattern_destroy(url_scanner->search_trie_full);
+			g_array_free(url_scanner->matchers_full, TRUE);
+		}
+
+		rspamd_multipattern_destroy(url_scanner->search_trie_strict);
+		g_array_free(url_scanner->matchers_strict, TRUE);
+		g_free(url_scanner);
+
+		url_scanner = NULL;
+	}
+}
+
+void rspamd_url_init(const gchar *tld_file)
+{
+	GError *err = NULL;
+	gboolean ret = TRUE;
+
+	if (url_scanner != NULL) {
+		rspamd_url_deinit();
+	}
+
+	url_scanner = g_malloc(sizeof(struct url_match_scanner));
+
+	url_scanner->matchers_strict = g_array_sized_new(FALSE, TRUE,
+													 sizeof(struct url_matcher), G_N_ELEMENTS(static_matchers));
+	url_scanner->search_trie_strict = rspamd_multipattern_create_sized(
+		G_N_ELEMENTS(static_matchers),
+		RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+
+	if (tld_file) {
+		/* Reserve larger multipattern */
+		url_scanner->matchers_full = g_array_sized_new(FALSE, TRUE,
+													   sizeof(struct url_matcher), 13000);
+		url_scanner->search_trie_full = rspamd_multipattern_create_sized(13000,
+																		 RSPAMD_MULTIPATTERN_ICASE | RSPAMD_MULTIPATTERN_UTF8);
+		url_scanner->has_tld_file = true;
+	}
+	else {
+		url_scanner->matchers_full = NULL;
+		url_scanner->search_trie_full = NULL;
+		url_scanner->has_tld_file = false;
+	}
+
+	rspamd_url_add_static_matchers(url_scanner);
+
+	if (tld_file != NULL) {
+		ret = rspamd_url_parse_tld_file(tld_file, url_scanner);
+	}
+
+	if (url_scanner->matchers_full && url_scanner->matchers_full->len > 1000) {
+		msg_info("start compiling of %d TLD suffixes; it might take a long time",
+				 url_scanner->matchers_full->len);
+	}
+
+	if (!rspamd_multipattern_compile(url_scanner->search_trie_strict, &err)) {
+		msg_err("cannot compile url matcher static patterns, fatal error: %e", err);
+		abort();
+	}
+
+	if (url_scanner->search_trie_full) {
+		if (!rspamd_multipattern_compile(url_scanner->search_trie_full, &err)) {
+			msg_err("cannot compile tld patterns, url matching will be "
+					"incomplete: %e",
+					err);
+			g_error_free(err);
+			ret = FALSE;
+		}
+	}
+
+	if (tld_file != NULL) {
+		if (ret) {
+			msg_info("initialized %ud url match suffixes from '%s'",
+					 url_scanner->matchers_full->len - url_scanner->matchers_strict->len,
+					 tld_file);
+		}
+		else {
+			msg_err("failed to initialize url tld suffixes from '%s', "
+					"use %ud internal match suffixes",
+					tld_file,
+					url_scanner->matchers_strict->len);
+		}
+	}
+
+	/* Generate hashes for flags */
+	for (gint i = 0; i < G_N_ELEMENTS(url_flag_names); i++) {
+		url_flag_names[i].hash =
+			rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT,
+												url_flag_names[i].name,
+												strlen(url_flag_names[i].name), 0);
+	}
+	/* Ensure that we have no hashes collisions O(N^2) but this array is small */
+	for (gint i = 0; i < G_N_ELEMENTS(url_flag_names) - 1; i++) {
+		for (gint j = i + 1; j < G_N_ELEMENTS(url_flag_names); j++) {
+			if (url_flag_names[i].hash == url_flag_names[j].hash) {
+				msg_err("collision: both %s and %s map to %d",
+						url_flag_names[i].name, url_flag_names[j].name,
+						url_flag_names[i].hash);
+				abort();
+			}
+		}
+	}
+}
+
+#define SET_U(u, field)                             \
+	do {                                            \
+		if ((u) != NULL) {                          \
+			(u)->field_set |= 1 << (field);         \
+			(u)->field_data[(field)].len = p - c;   \
+			(u)->field_data[(field)].off = c - str; \
+		}                                           \
+	} while (0)
+
+static bool
+is_url_start(gchar c)
+{
+	if (c == '(' ||
+		c == '{' ||
+		c == '[' ||
+		c == '<' ||
+		c == '\'') {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static bool
+is_url_end(gchar c)
+{
+	if (c == ')' ||
+		c == '}' ||
+		c == ']' ||
+		c == '>' ||
+		c == '\'') {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static bool
+is_domain_start(int p)
+{
+	if (g_ascii_isalnum(p) ||
+		p == '[' ||
+		p == '%' ||
+		p == '_' ||
+		(p & 0x80)) {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static const guint max_domain_length = 253;
+static const guint max_dns_label = 63;
+static const guint max_email_user = 64;
+
+static gint
+rspamd_mailto_parse(struct http_parser_url *u,
+					const gchar *str, gsize len,
+					gchar const **end,
+					enum rspamd_url_parse_flags parse_flags, guint *flags)
+{
+	const gchar *p = str, *c = str, *last = str + len;
+	gchar t;
+	gint ret = 1;
+	enum {
+		parse_mailto,
+		parse_slash,
+		parse_slash_slash,
+		parse_semicolon,
+		parse_prefix_question,
+		parse_destination,
+		parse_equal,
+		parse_user,
+		parse_at,
+		parse_domain,
+		parse_suffix_question,
+		parse_query
+	} st = parse_mailto;
+
+	if (u != NULL) {
+		memset(u, 0, sizeof(*u));
+	}
+
+	while (p < last) {
+		t = *p;
+
+		if (p - str > max_email_user + max_domain_length + 1) {
+			goto out;
+		}
+
+		switch (st) {
+		case parse_mailto:
+			if (t == ':') {
+				st = parse_semicolon;
+				SET_U(u, UF_SCHEMA);
+			}
+			p++;
+			break;
+		case parse_semicolon:
+			if (t == '/' || t == '\\') {
+				st = parse_slash;
+				p++;
+			}
+			else {
+				*flags |= RSPAMD_URL_FLAG_MISSINGSLASHES;
+				st = parse_slash_slash;
+			}
+			break;
+		case parse_slash:
+			if (t == '/' || t == '\\') {
+				st = parse_slash_slash;
+			}
+			else {
+				goto out;
+			}
+			p++;
+			break;
+		case parse_slash_slash:
+			if (t == '?') {
+				st = parse_prefix_question;
+				p++;
+			}
+			else if (t != '/' && t != '\\') {
+				c = p;
+				st = parse_user;
+			}
+			else {
+				/* Skip multiple slashes */
+				p++;
+			}
+			break;
+		case parse_prefix_question:
+			if (t == 't') {
+				/* XXX: accept only to= */
+				st = parse_destination;
+			}
+			else {
+				goto out;
+			}
+			break;
+		case parse_destination:
+			if (t == '=') {
+				st = parse_equal;
+			}
+			p++;
+			break;
+		case parse_equal:
+			c = p;
+			st = parse_user;
+			break;
+		case parse_user:
+			if (t == '@') {
+				if (p - c == 0) {
+					goto out;
+				}
+				SET_U(u, UF_USERINFO);
+				st = parse_at;
+			}
+			else if (!is_mailsafe(t)) {
+				goto out;
+			}
+			else if (p - c > max_email_user) {
+				goto out;
+			}
+			p++;
+			break;
+		case parse_at:
+			c = p;
+			st = parse_domain;
+			break;
+		case parse_domain:
+			if (t == '?') {
+				SET_U(u, UF_HOST);
+				st = parse_suffix_question;
+			}
+			else if (!is_domain(t) && t != '.' && t != '_') {
+				goto out;
+			}
+			else if (p - c > max_domain_length) {
+				goto out;
+			}
+			p++;
+			break;
+		case parse_suffix_question:
+			c = p;
+			st = parse_query;
+			break;
+		case parse_query:
+			if (t == '#') {
+				if (p - c != 0) {
+					SET_U(u, UF_QUERY);
+				}
+				c = p + 1;
+				ret = 0;
+
+				goto out;
+			}
+			else if (!(parse_flags & RSPAMD_URL_PARSE_HREF) && is_url_end(t)) {
+				ret = 0;
+				goto out;
+			}
+			else if (is_lwsp(t)) {
+				if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+					if (g_ascii_isspace(t)) {
+						ret = 0;
+					}
+					goto out;
+				}
+				else {
+					goto out;
+				}
+			}
+			p++;
+			break;
+		}
+	}
+
+	if (st == parse_domain) {
+		if (p - c != 0) {
+			SET_U(u, UF_HOST);
+			ret = 0;
+		}
+	}
+	else if (st == parse_query) {
+		if (p - c > 0) {
+			SET_U(u, UF_QUERY);
+		}
+
+		ret = 0;
+	}
+
+out:
+	if (end != NULL) {
+		*end = p;
+	}
+
+	if ((parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+		return 0;
+	}
+
+	return ret;
+}
+
+static gint
+rspamd_telephone_parse(struct http_parser_url *u,
+					   const gchar *str, gsize len,
+					   gchar const **end,
+					   enum rspamd_url_parse_flags parse_flags,
+					   guint *flags)
+{
+	enum {
+		parse_protocol,
+		parse_semicolon,
+		parse_slash,
+		parse_slash_slash,
+		parse_spaces,
+		parse_plus,
+		parse_phone_start,
+		parse_phone,
+	} st = parse_protocol;
+
+	const gchar *p = str, *c = str, *last = str + len;
+	gchar t;
+	gint ret = 1, i;
+	UChar32 uc;
+
+	if (u != NULL) {
+		memset(u, 0, sizeof(*u));
+	}
+
+	while (p < last) {
+		t = *p;
+
+		if (p - str > max_email_user) {
+			goto out;
+		}
+
+		switch (st) {
+		case parse_protocol:
+			if (t == ':') {
+				st = parse_semicolon;
+				SET_U(u, UF_SCHEMA);
+			}
+			p++;
+			break;
+		case parse_semicolon:
+			if (t == '/' || t == '\\') {
+				st = parse_slash;
+				p++;
+			}
+			else {
+				st = parse_slash_slash;
+			}
+			break;
+		case parse_slash:
+			if (t == '/' || t == '\\') {
+				st = parse_slash_slash;
+			}
+			else {
+				goto out;
+			}
+			p++;
+			break;
+		case parse_slash_slash:
+			if (g_ascii_isspace(t)) {
+				st = parse_spaces;
+				p++;
+			}
+			else if (t == '+') {
+				c = p;
+				st = parse_plus;
+			}
+			else if (t == '/') {
+				/* Skip multiple slashes */
+				p++;
+			}
+			else {
+				st = parse_phone_start;
+				c = p;
+			}
+			break;
+		case parse_spaces:
+			if (t == '+') {
+				c = p;
+				st = parse_plus;
+			}
+			else if (!g_ascii_isspace(t)) {
+				st = parse_phone_start;
+				c = p;
+			}
+			else {
+				p++;
+			}
+			break;
+		case parse_plus:
+			c = p;
+			p++;
+			st = parse_phone_start;
+			break;
+		case parse_phone_start:
+			if (*p == '%' || *p == '(' || g_ascii_isdigit(*p)) {
+				st = parse_phone;
+				p++;
+			}
+			else {
+				goto out;
+			}
+			break;
+		case parse_phone:
+			i = p - str;
+			U8_NEXT(str, i, len, uc);
+			p = str + i;
+
+			if (u_isdigit(uc) || uc == '(' || uc == ')' || uc == '[' || uc == ']' || u_isspace(uc) || uc == '%') {
+				/* p is already incremented by U8_NEXT! */
+			}
+			else if (uc <= 0 || is_url_end(uc)) {
+				ret = 0;
+				goto set;
+			}
+			break;
+		}
+	}
+
+set:
+	if (st == parse_phone) {
+		if (p - c != 0) {
+			SET_U(u, UF_HOST);
+			ret = 0;
+		}
+	}
+
+out:
+	if (end != NULL) {
+		*end = p;
+	}
+
+	if ((parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+		return 0;
+	}
+
+	return ret;
+}
+
+static gint
+rspamd_web_parse(struct http_parser_url *u, const gchar *str, gsize len,
+				 gchar const **end,
+				 enum rspamd_url_parse_flags parse_flags,
+				 guint *flags)
+{
+	const gchar *p = str, *c = str, *last = str + len, *slash = NULL,
+				*password_start = NULL, *user_start = NULL;
+	gchar t = 0;
+	UChar32 uc;
+	glong pt;
+	gint ret = 1;
+	gboolean user_seen = FALSE;
+	enum {
+		parse_protocol,
+		parse_slash,
+		parse_slash_slash,
+		parse_semicolon,
+		parse_user,
+		parse_at,
+		parse_multiple_at,
+		parse_password_start,
+		parse_password,
+		parse_domain_start,
+		parse_domain,
+		parse_ipv6,
+		parse_port_password,
+		parse_port,
+		parse_suffix_slash,
+		parse_path,
+		parse_query,
+		parse_part
+	} st = parse_protocol;
+
+	if (u != NULL) {
+		memset(u, 0, sizeof(*u));
+	}
+
+	while (p < last) {
+		t = *p;
+
+		switch (st) {
+		case parse_protocol:
+			if (t == ':') {
+				st = parse_semicolon;
+				SET_U(u, UF_SCHEMA);
+			}
+			else if (!g_ascii_isalnum(t) && t != '+' && t != '-') {
+				if ((parse_flags & RSPAMD_URL_PARSE_CHECK) && p > c) {
+					/* We might have some domain, but no protocol */
+					st = parse_domain_start;
+					p = c;
+					slash = c;
+					break;
+				}
+				else {
+					goto out;
+				}
+			}
+			p++;
+			break;
+		case parse_semicolon:
+			if (t == '/' || t == '\\') {
+				st = parse_slash;
+				p++;
+			}
+			else {
+				st = parse_slash_slash;
+				*(flags) |= RSPAMD_URL_FLAG_MISSINGSLASHES;
+			}
+			break;
+		case parse_slash:
+			if (t == '/' || t == '\\') {
+				st = parse_slash_slash;
+			}
+			else {
+				goto out;
+			}
+			p++;
+			break;
+		case parse_slash_slash:
+
+			if (t != '/' && t != '\\') {
+				c = p;
+				slash = p;
+				st = parse_domain_start;
+
+				/*
+				 * Unfortunately, due to brain damage of the RFC 3986 authors,
+				 * we have to distinguish two possibilities here:
+				 * authority = [ userinfo "@" ] host [ ":" port ]
+				 * So if we have @ somewhere before hostname then we must process
+				 * with the username state. Otherwise, we have to process via
+				 * the hostname state. Unfortunately, there is no way to distinguish
+				 * them aside of running NFA or two DFA or performing lookahead.
+				 * Lookahead approach looks easier to implement.
+				 */
+
+				const char *tp = p;
+				while (tp < last) {
+					if (*tp == '@') {
+						user_seen = TRUE;
+						st = parse_user;
+						break;
+					}
+					else if (*tp == '/' || *tp == '#' || *tp == '?') {
+						st = parse_domain_start;
+						break;
+					}
+
+					tp++;
+				}
+
+				if (st == parse_domain_start && *p == '[') {
+					st = parse_ipv6;
+					p++;
+					c = p;
+				}
+			}
+			else {
+				/* Skip multiple slashes */
+				p++;
+			}
+			break;
+		case parse_ipv6:
+			if (t == ']') {
+				if (p - c == 0) {
+					goto out;
+				}
+				SET_U(u, UF_HOST);
+				p++;
+
+				if (*p == ':') {
+					st = parse_port;
+					c = p + 1;
+				}
+				else if (*p == '/' || *p == '\\') {
+					st = parse_path;
+					c = p + 1;
+				}
+				else if (*p == '?') {
+					st = parse_query;
+					c = p + 1;
+				}
+				else if (*p == '#') {
+					st = parse_part;
+					c = p + 1;
+				}
+				else if (p != last) {
+					goto out;
+				}
+			}
+			else if (!g_ascii_isxdigit(t) && t != ':' && t != '.') {
+				goto out;
+			}
+			p++;
+			break;
+		case parse_user:
+			if (t == ':') {
+				if (p - c == 0) {
+					goto out;
+				}
+				user_start = c;
+				st = parse_password_start;
+			}
+			else if (t == '@') {
+				/* No password */
+				if (p - c == 0) {
+					/* We have multiple at in fact */
+					st = parse_multiple_at;
+					user_seen = TRUE;
+					*flags |= RSPAMD_URL_FLAG_OBSCURED;
+
+					continue;
+				}
+
+				SET_U(u, UF_USERINFO);
+				*flags |= RSPAMD_URL_FLAG_HAS_USER;
+				st = parse_at;
+			}
+			else if (!g_ascii_isgraph(t)) {
+				goto out;
+			}
+			else if (p - c > max_email_user) {
+				goto out;
+			}
+
+			p++;
+			break;
+		case parse_multiple_at:
+			if (t != '@') {
+				if (p - c == 0) {
+					goto out;
+				}
+
+				/* For now, we ignore all that stuff as it is bogus */
+				/* Off by one */
+				p--;
+				SET_U(u, UF_USERINFO);
+				p++;
+				*flags |= RSPAMD_URL_FLAG_HAS_USER;
+				st = parse_at;
+			}
+			else {
+				p++;
+			}
+			break;
+		case parse_password_start:
+			if (t == '@') {
+				/* Empty password */
+				SET_U(u, UF_USERINFO);
+				if (u != NULL && u->field_data[UF_USERINFO].len > 0) {
+					/* Eat semicolon */
+					u->field_data[UF_USERINFO].len--;
+				}
+				*flags |= RSPAMD_URL_FLAG_HAS_USER;
+				st = parse_at;
+			}
+			else {
+				c = p;
+				password_start = p;
+				st = parse_password;
+			}
+			p++;
+			break;
+		case parse_password:
+			if (t == '@') {
+				/* XXX: password is not stored */
+				if (u != NULL) {
+					if (u->field_data[UF_USERINFO].len == 0 && password_start && user_start && password_start > user_start + 1) {
+						*flags |= RSPAMD_URL_FLAG_HAS_USER;
+						u->field_set |= 1u << (UF_USERINFO);
+						u->field_data[UF_USERINFO].len =
+							password_start - user_start - 1;
+						u->field_data[UF_USERINFO].off =
+							user_start - str;
+					}
+				}
+				st = parse_at;
+			}
+			else if (!g_ascii_isgraph(t)) {
+				goto out;
+			}
+			else if (p - c > max_domain_length) {
+				goto out;
+			}
+			p++;
+			break;
+		case parse_at:
+			c = p;
+
+			if (t == '@') {
+				*flags |= RSPAMD_URL_FLAG_OBSCURED;
+				p++;
+			}
+			else if (t == '[') {
+				st = parse_ipv6;
+				p++;
+				c = p;
+			}
+			else {
+				st = parse_domain_start;
+			}
+			break;
+		case parse_domain_start:
+			if (is_domain_start(t)) {
+				st = parse_domain;
+			}
+			else {
+				goto out;
+			}
+			break;
+		case parse_domain:
+			if (p - c > max_domain_length) {
+				/* Too large domain */
+				goto out;
+			}
+			if (t == '/' || t == '\\' || t == ':' || t == '?' || t == '#') {
+				if (p - c == 0) {
+					goto out;
+				}
+				if (t == '/' || t == '\\') {
+					SET_U(u, UF_HOST);
+					st = parse_suffix_slash;
+				}
+				else if (t == '?') {
+					SET_U(u, UF_HOST);
+					st = parse_query;
+					c = p + 1;
+				}
+				else if (t == '#') {
+					SET_U(u, UF_HOST);
+					st = parse_part;
+					c = p + 1;
+				}
+				else if (t == ':' && !user_seen) {
+					/*
+					 * Here we can have both port and password, hence we need
+					 * to apply some heuristic here
+					 */
+					st = parse_port_password;
+				}
+				else {
+					/*
+					 * We can go only for parsing port here
+					 */
+					SET_U(u, UF_HOST);
+					st = parse_port;
+					c = p + 1;
+				}
+				p++;
+			}
+			else {
+				if (is_url_end(t) || is_url_start(t)) {
+					goto set;
+				}
+				else if (*p == '@' && !user_seen) {
+					/* We need to fallback and test user */
+					p = slash;
+					user_seen = TRUE;
+					st = parse_user;
+				}
+				else if (*p != '.' && *p != '-' && *p != '_' && *p != '%') {
+					if (*p & 0x80) {
+						guint i = 0;
+
+						U8_NEXT(((const guchar *) p), i, last - p, uc);
+
+						if (uc < 0) {
+							/* Bad utf8 */
+							goto out;
+						}
+
+						if (!u_isalnum(uc)) {
+							/* Bad symbol */
+							if (IS_ZERO_WIDTH_SPACE(uc)) {
+								(*flags) |= RSPAMD_URL_FLAG_ZW_SPACES;
+							}
+							else {
+								if (!u_isgraph(uc)) {
+									if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+										goto out;
+									}
+									else {
+										goto set;
+									}
+								}
+							}
+						}
+						else {
+							(*flags) |= RSPAMD_URL_FLAG_IDN;
+						}
+
+						p = p + i;
+					}
+					else if (is_urlsafe(*p)) {
+						p++;
+					}
+					else {
+						if (parse_flags & RSPAMD_URL_PARSE_HREF) {
+							/* We have to use all shit we are given here */
+							p++;
+							(*flags) |= RSPAMD_URL_FLAG_OBSCURED;
+						}
+						else {
+							if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+								goto out;
+							}
+							else {
+								goto set;
+							}
+						}
+					}
+				}
+				else {
+					p++;
+				}
+			}
+			break;
+		case parse_port_password:
+			if (g_ascii_isdigit(t)) {
+				const gchar *tmp = p;
+
+				while (tmp < last) {
+					if (!g_ascii_isdigit(*tmp)) {
+						if (*tmp == '/' || *tmp == '#' || *tmp == '?' ||
+							is_url_end(*tmp) || g_ascii_isspace(*tmp)) {
+							/* Port + something */
+							st = parse_port;
+							c = slash;
+							p--;
+							SET_U(u, UF_HOST);
+							p++;
+							c = p;
+							break;
+						}
+						else {
+							/* Not a port, bad character at the end */
+							break;
+						}
+					}
+					tmp++;
+				}
+
+				if (tmp == last) {
+					/* Host + port only */
+					st = parse_port;
+					c = slash;
+					p--;
+					SET_U(u, UF_HOST);
+					p++;
+					c = p;
+				}
+
+				if (st != parse_port) {
+					/* Fallback to user:password */
+					p = slash;
+					c = slash;
+					user_seen = TRUE;
+					st = parse_user;
+				}
+			}
+			else {
+				/* Rewind back */
+				p = slash;
+				c = slash;
+				user_seen = TRUE;
+				st = parse_user;
+			}
+			break;
+		case parse_port:
+			if (t == '/' || t == '\\') {
+				pt = strtoul(c, NULL, 10);
+				if (pt == 0 || pt > 65535) {
+					goto out;
+				}
+				if (u != NULL) {
+					u->port = pt;
+					*flags |= RSPAMD_URL_FLAG_HAS_PORT;
+				}
+				st = parse_suffix_slash;
+			}
+			else if (t == '?') {
+				pt = strtoul(c, NULL, 10);
+				if (pt == 0 || pt > 65535) {
+					goto out;
+				}
+				if (u != NULL) {
+					u->port = pt;
+					*flags |= RSPAMD_URL_FLAG_HAS_PORT;
+				}
+
+				c = p + 1;
+				st = parse_query;
+			}
+			else if (t == '#') {
+				pt = strtoul(c, NULL, 10);
+				if (pt == 0 || pt > 65535) {
+					goto out;
+				}
+				if (u != NULL) {
+					u->port = pt;
+					*flags |= RSPAMD_URL_FLAG_HAS_PORT;
+				}
+
+				c = p + 1;
+				st = parse_part;
+			}
+			else if (is_url_end(t)) {
+				goto set;
+			}
+			else if (!g_ascii_isdigit(t)) {
+				if (!(parse_flags & RSPAMD_URL_PARSE_CHECK) ||
+					!g_ascii_isspace(t)) {
+					goto out;
+				}
+				else {
+					goto set;
+				}
+			}
+			p++;
+			break;
+		case parse_suffix_slash:
+			if (t != '/' && t != '\\') {
+				c = p;
+				st = parse_path;
+			}
+			else {
+				/* Skip extra slashes */
+				p++;
+			}
+			break;
+		case parse_path:
+			if (t == '?') {
+				if (p - c != 0) {
+					SET_U(u, UF_PATH);
+				}
+				c = p + 1;
+				st = parse_query;
+			}
+			else if (t == '#') {
+				/* No query, just fragment */
+				if (p - c != 0) {
+					SET_U(u, UF_PATH);
+				}
+				c = p + 1;
+				st = parse_part;
+			}
+			else if (!(parse_flags & RSPAMD_URL_PARSE_HREF) && is_url_end(t)) {
+				goto set;
+			}
+			else if (is_lwsp(t)) {
+				if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+					if (g_ascii_isspace(t)) {
+						goto set;
+					}
+					goto out;
+				}
+				else {
+					goto set;
+				}
+			}
+			p++;
+			break;
+		case parse_query:
+			if (t == '#') {
+				if (p - c != 0) {
+					SET_U(u, UF_QUERY);
+				}
+				c = p + 1;
+				st = parse_part;
+			}
+			else if (!(parse_flags & RSPAMD_URL_PARSE_HREF) && is_url_end(t)) {
+				goto set;
+			}
+			else if (is_lwsp(t)) {
+				if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+					if (g_ascii_isspace(t)) {
+						goto set;
+					}
+					goto out;
+				}
+				else {
+					goto set;
+				}
+			}
+			p++;
+			break;
+		case parse_part:
+			if (!(parse_flags & RSPAMD_URL_PARSE_HREF) && is_url_end(t)) {
+				goto set;
+			}
+			else if (is_lwsp(t)) {
+				if (!(parse_flags & RSPAMD_URL_PARSE_CHECK)) {
+					if (g_ascii_isspace(t)) {
+						goto set;
+					}
+					goto out;
+				}
+				else {
+					goto set;
+				}
+			}
+			p++;
+			break;
+		}
+	}
+
+set:
+	/* Parse remaining */
+	switch (st) {
+	case parse_domain:
+		if (p - c == 0 || !is_domain(*(p - 1)) || !is_domain(*c)) {
+			goto out;
+		}
+		SET_U(u, UF_HOST);
+		ret = 0;
+
+		break;
+	case parse_port:
+		pt = strtoul(c, NULL, 10);
+		if (pt == 0 || pt > 65535) {
+			goto out;
+		}
+		if (u != NULL) {
+			u->port = pt;
+		}
+
+		ret = 0;
+		break;
+	case parse_suffix_slash:
+		/* Url ends with '/' */
+		ret = 0;
+		break;
+	case parse_path:
+		if (p - c > 0) {
+			SET_U(u, UF_PATH);
+		}
+		ret = 0;
+		break;
+	case parse_query:
+		if (p - c > 0) {
+			SET_U(u, UF_QUERY);
+		}
+		ret = 0;
+		break;
+	case parse_part:
+		if (p - c > 0) {
+			SET_U(u, UF_FRAGMENT);
+		}
+		ret = 0;
+		break;
+	case parse_ipv6:
+		if (t != ']') {
+			ret = 1;
+		}
+		else {
+			/* e.g. http://[::] */
+			ret = 0;
+		}
+		break;
+	default:
+		/* Error state */
+		ret = 1;
+		break;
+	}
+out:
+	if (end != NULL) {
+		*end = p;
+	}
+
+	return ret;
+}
+
+#undef SET_U
+
+static gint
+rspamd_tld_trie_callback(struct rspamd_multipattern *mp,
+						 guint strnum,
+						 gint match_start,
+						 gint match_pos,
+						 const gchar *text,
+						 gsize len,
+						 void *context)
+{
+	struct url_matcher *matcher;
+	const gchar *start, *pos, *p;
+	struct rspamd_url *url = context;
+	gint ndots;
+
+	matcher = &g_array_index(url_scanner->matchers_full, struct url_matcher,
+							 strnum);
+	ndots = 1;
+
+	if (matcher->flags & URL_MATCHER_FLAG_STAR_MATCH) {
+		/* Skip one more tld component */
+		ndots++;
+	}
+
+	pos = text + match_start;
+	p = pos - 1;
+	start = rspamd_url_host_unsafe(url);
+
+	if (*pos != '.' || match_pos != (gint) url->hostlen) {
+		/* Something weird has been found */
+		if (match_pos == (gint) url->hostlen - 1) {
+			pos = rspamd_url_host_unsafe(url) + match_pos;
+			if (*pos == '.') {
+				/* This is dot at the end of domain */
+				url->hostlen--;
+			}
+			else {
+				return 0;
+			}
+		}
+		else {
+			return 0;
+		}
+	}
+
+	/* Now we need to find top level domain */
+	pos = start;
+	while (p >= start && ndots > 0) {
+		if (*p == '.') {
+			ndots--;
+			pos = p + 1;
+		}
+		else {
+			pos = p;
+		}
+
+		p--;
+	}
+
+	if ((ndots == 0 || p == start - 1) &&
+		url->tldlen < rspamd_url_host_unsafe(url) + url->hostlen - pos) {
+		url->tldshift = (pos - url->string);
+		url->tldlen = rspamd_url_host_unsafe(url) + url->hostlen - pos;
+	}
+
+	return 0;
+}
+
+static void
+rspamd_url_regen_from_inet_addr(struct rspamd_url *uri, const void *addr, int af,
+								rspamd_mempool_t *pool)
+{
+	gchar *strbuf, *p;
+	const gchar *start_offset;
+	gsize slen = uri->urllen - uri->hostlen;
+	goffset r = 0;
+
+	if (af == AF_INET) {
+		slen += INET_ADDRSTRLEN;
+	}
+	else {
+		slen += INET6_ADDRSTRLEN;
+	}
+
+	if (uri->flags & RSPAMD_URL_FLAG_HAS_PORT) {
+		slen += sizeof("65535") - 1;
+	}
+
+	/* Allocate new string to build it from IP */
+	strbuf = rspamd_mempool_alloc(pool, slen + 1);
+	r += rspamd_snprintf(strbuf + r, slen - r, "%*s",
+						 (gint) (uri->hostshift),
+						 uri->string);
+
+	uri->hostshift = r;
+	uri->tldshift = r;
+	start_offset = strbuf + r;
+	inet_ntop(af, addr, strbuf + r, slen - r + 1);
+	uri->hostlen = strlen(start_offset);
+	r += uri->hostlen;
+	uri->tldlen = uri->hostlen;
+	uri->flags |= RSPAMD_URL_FLAG_NUMERIC;
+
+	/* Reconstruct URL */
+	if (uri->flags & RSPAMD_URL_FLAG_HAS_PORT && uri->ext) {
+		p = strbuf + r;
+		start_offset = p + 1;
+		r += rspamd_snprintf(strbuf + r, slen - r, ":%ud",
+							 (unsigned int) uri->ext->port);
+	}
+	if (uri->datalen > 0) {
+		p = strbuf + r;
+		start_offset = p + 1;
+		r += rspamd_snprintf(strbuf + r, slen - r, "/%*s",
+							 (gint) uri->datalen,
+							 rspamd_url_data_unsafe(uri));
+		uri->datashift = start_offset - strbuf;
+	}
+	else {
+		/* Add trailing slash if needed */
+		if (uri->hostlen + uri->hostshift < uri->urllen &&
+			*(rspamd_url_host_unsafe(uri) + uri->hostlen) == '/') {
+			r += rspamd_snprintf(strbuf + r, slen - r, "/");
+		}
+	}
+
+	if (uri->querylen > 0) {
+		p = strbuf + r;
+		start_offset = p + 1;
+		r += rspamd_snprintf(strbuf + r, slen - r, "?%*s",
+							 (gint) uri->querylen,
+							 rspamd_url_query_unsafe(uri));
+		uri->queryshift = start_offset - strbuf;
+	}
+	if (uri->fragmentlen > 0) {
+		p = strbuf + r;
+		start_offset = p + 1;
+		r += rspamd_snprintf(strbuf + r, slen - r, "#%*s",
+							 (gint) uri->fragmentlen,
+							 rspamd_url_fragment_unsafe(uri));
+		uri->fragmentshift = start_offset - strbuf;
+	}
+
+	uri->string = strbuf;
+	uri->urllen = r;
+}
+
+static gboolean
+rspamd_url_maybe_regenerate_from_ip(struct rspamd_url *uri, rspamd_mempool_t *pool)
+{
+	const gchar *p, *end, *c;
+	gchar *errstr;
+	struct in_addr in4;
+	struct in6_addr in6;
+	gboolean ret = FALSE, check_num = TRUE;
+	guint32 n, dots, t = 0, i = 0, shift, nshift;
+
+	p = rspamd_url_host_unsafe(uri);
+	end = p + uri->hostlen;
+
+	if (*p == '[' && *(end - 1) == ']') {
+		p++;
+		end--;
+	}
+
+	while (*(end - 1) == '.' && end > p) {
+		end--;
+	}
+
+	if (end - p == 0 || end - p > INET6_ADDRSTRLEN) {
+		return FALSE;
+	}
+
+	if (rspamd_str_has_8bit(p, end - p)) {
+		return FALSE;
+	}
+
+	if (rspamd_parse_inet_address_ip4(p, end - p, &in4)) {
+		rspamd_url_regen_from_inet_addr(uri, &in4, AF_INET, pool);
+		ret = TRUE;
+	}
+	else if (rspamd_parse_inet_address_ip6(p, end - p, &in6)) {
+		rspamd_url_regen_from_inet_addr(uri, &in6, AF_INET6, pool);
+		ret = TRUE;
+	}
+	else {
+		/* Heuristics for broken urls */
+		gchar buf[INET6_ADDRSTRLEN + 1];
+		/* Try also numeric notation */
+		c = p;
+		n = 0;
+		dots = 0;
+		shift = 0;
+
+		while (p <= end && check_num) {
+			if (shift < 32 &&
+				((*p == '.' && dots < 3) || (p == end && dots <= 3))) {
+				if (p - c + 1 >= (gint) sizeof(buf)) {
+					msg_debug_pool("invalid numeric url %*.s...: too long",
+								   INET6_ADDRSTRLEN, c);
+					return FALSE;
+				}
+
+				rspamd_strlcpy(buf, c, p - c + 1);
+				c = p + 1;
+
+				if (p < end && *p == '.') {
+					dots++;
+				}
+
+				glong long_n = strtol(buf, &errstr, 0);
+
+				if ((errstr == NULL || *errstr == '\0') && long_n >= 0) {
+
+					t = long_n; /* Truncate as windows does */
+					/*
+					 * Even if we have zero, we need to shift by 1 octet
+					 */
+					nshift = (t == 0 ? shift + 8 : shift);
+
+					/*
+					 * Here we count number of octets encoded in this element
+					 */
+					for (i = 0; i < 4; i++) {
+						if ((t >> (8 * i)) > 0) {
+							nshift += 8;
+						}
+						else {
+							break;
+						}
+					}
+					/*
+					 * Here we need to find the proper shift of the previous
+					 * components, so we check possible cases:
+					 * 1) 1 octet - just use it applying shift
+					 * 2) 2 octets - convert to big endian 16 bit number
+					 * 3) 3 octets - convert to big endian 24 bit number
+					 * 4) 4 octets - convert to big endian 32 bit number
+					 */
+					switch (i) {
+					case 4:
+						t = GUINT32_TO_BE(t);
+						break;
+					case 3:
+						t = (GUINT32_TO_BE(t & 0xFFFFFFU)) >> 8;
+						break;
+					case 2:
+						t = GUINT16_TO_BE(t & 0xFFFFU);
+						break;
+					default:
+						t = t & 0xFF;
+						break;
+					}
+
+					if (p != end) {
+						n |= t << shift;
+
+						shift = nshift;
+					}
+				}
+				else {
+					check_num = FALSE;
+				}
+			}
+
+			p++;
+		}
+
+		/* The last component should be last according to url normalization:
+		 * 192.168.1 -> 192.168.0.1
+		 * 192 -> 0.0.0.192
+		 * 192.168 -> 192.0.0.168
+		 */
+		shift = 8 * (4 - i);
+
+		if (shift < 32) {
+			n |= t << shift;
+		}
+
+		if (check_num) {
+			if (dots <= 4) {
+				memcpy(&in4, &n, sizeof(in4));
+				rspamd_url_regen_from_inet_addr(uri, &in4, AF_INET, pool);
+				uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+				ret = TRUE;
+			}
+			else if (end - c > (gint) sizeof(buf) - 1) {
+				rspamd_strlcpy(buf, c, end - c + 1);
+
+				if (inet_pton(AF_INET6, buf, &in6) == 1) {
+					rspamd_url_regen_from_inet_addr(uri, &in6, AF_INET6, pool);
+					uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+					ret = TRUE;
+				}
+			}
+		}
+	}
+
+	return ret;
+}
+
+static void
+rspamd_url_shift(struct rspamd_url *uri, gsize nlen,
+				 enum http_parser_url_fields field)
+{
+	guint old_shift, shift = 0;
+	gint remain;
+
+	/* Shift remaining data */
+	switch (field) {
+	case UF_SCHEMA:
+		if (nlen >= uri->protocollen) {
+			return;
+		}
+		else {
+			shift = uri->protocollen - nlen;
+		}
+
+		old_shift = uri->protocollen;
+		uri->protocollen -= shift;
+		remain = uri->urllen - uri->protocollen;
+		g_assert(remain >= 0);
+		memmove(uri->string + uri->protocollen, uri->string + old_shift,
+				remain);
+		uri->urllen -= shift;
+		uri->flags |= RSPAMD_URL_FLAG_SCHEMAENCODED;
+		break;
+	case UF_HOST:
+		if (nlen >= uri->hostlen) {
+			return;
+		}
+		else {
+			shift = uri->hostlen - nlen;
+		}
+
+		old_shift = uri->hostlen;
+		uri->hostlen -= shift;
+		remain = (uri->urllen - (uri->hostshift)) - old_shift;
+		g_assert(remain >= 0);
+		memmove(rspamd_url_host_unsafe(uri) + uri->hostlen,
+				rspamd_url_host_unsafe(uri) + old_shift,
+				remain);
+		uri->urllen -= shift;
+		uri->flags |= RSPAMD_URL_FLAG_HOSTENCODED;
+		break;
+	case UF_PATH:
+		if (nlen >= uri->datalen) {
+			return;
+		}
+		else {
+			shift = uri->datalen - nlen;
+		}
+
+		old_shift = uri->datalen;
+		uri->datalen -= shift;
+		remain = (uri->urllen - (uri->datashift)) - old_shift;
+		g_assert(remain >= 0);
+		memmove(rspamd_url_data_unsafe(uri) + uri->datalen,
+				rspamd_url_data_unsafe(uri) + old_shift,
+				remain);
+		uri->urllen -= shift;
+		uri->flags |= RSPAMD_URL_FLAG_PATHENCODED;
+		break;
+	case UF_QUERY:
+		if (nlen >= uri->querylen) {
+			return;
+		}
+		else {
+			shift = uri->querylen - nlen;
+		}
+
+		old_shift = uri->querylen;
+		uri->querylen -= shift;
+		remain = (uri->urllen - (uri->queryshift)) - old_shift;
+		g_assert(remain >= 0);
+		memmove(rspamd_url_query_unsafe(uri) + uri->querylen,
+				rspamd_url_query_unsafe(uri) + old_shift,
+				remain);
+		uri->urllen -= shift;
+		uri->flags |= RSPAMD_URL_FLAG_QUERYENCODED;
+		break;
+	case UF_FRAGMENT:
+		if (nlen >= uri->fragmentlen) {
+			return;
+		}
+		else {
+			shift = uri->fragmentlen - nlen;
+		}
+
+		uri->fragmentlen -= shift;
+		uri->urllen -= shift;
+		break;
+	default:
+		break;
+	}
+
+	/* Now adjust lengths and offsets */
+	switch (field) {
+	case UF_SCHEMA:
+		if (uri->userlen > 0) {
+			uri->usershift -= shift;
+		}
+		if (uri->hostlen > 0) {
+			uri->hostshift -= shift;
+		}
+		/* Go forward */
+		/* FALLTHRU */
+	case UF_HOST:
+		if (uri->datalen > 0) {
+			uri->datashift -= shift;
+		}
+		/* Go forward */
+		/* FALLTHRU */
+	case UF_PATH:
+		if (uri->querylen > 0) {
+			uri->queryshift -= shift;
+		}
+		/* Go forward */
+		/* FALLTHRU */
+	case UF_QUERY:
+		if (uri->fragmentlen > 0) {
+			uri->fragmentshift -= shift;
+		}
+		/* Go forward */
+		/* FALLTHRU */
+	case UF_FRAGMENT:
+	default:
+		break;
+	}
+}
+
+static void
+rspamd_telephone_normalise_inplace(struct rspamd_url *uri)
+{
+	gchar *t, *h, *end;
+	gint i = 0, w, orig_len;
+	UChar32 uc;
+
+	t = rspamd_url_host_unsafe(uri);
+	h = t;
+	end = t + uri->hostlen;
+	orig_len = uri->hostlen;
+
+	if (*h == '+') {
+		h++;
+		t++;
+	}
+
+	while (h < end) {
+		i = 0;
+		U8_NEXT(h, i, end - h, uc);
+
+		if (u_isdigit(uc)) {
+			w = 0;
+			U8_APPEND_UNSAFE(t, w, uc);
+			t += w;
+		}
+
+		h += i;
+	}
+
+	uri->hostlen = t - rspamd_url_host_unsafe(uri);
+	uri->urllen -= (orig_len - uri->hostlen);
+}
+
+static inline bool
+is_idna_label_dot(UChar ch)
+{
+	switch (ch) {
+	case 0x3002:
+	case 0xFF0E:
+	case 0xFF61:
+		return true;
+	default:
+		return false;
+	}
+}
+
+/*
+ * All credits for this investigation should go to
+ * Dr. Hajime Shimada and Mr. Shirakura as they have revealed this case in their
+ * research.
+ */
+
+/*
+ * This function replaces unsafe IDNA dots in host labels. Unfortunately,
+ * IDNA extends dot definition from '.' to multiple other characters that
+ * should be treated equally.
+ * This function replaces such dots and returns `true` if these dots are found.
+ * In this case, it should be treated as obfuscation attempt.
+ */
+static bool
+rspamd_url_remove_dots(struct rspamd_url *uri)
+{
+	const gchar *hstart = rspamd_url_host_unsafe(uri);
+	gchar *t;
+	UChar32 uc;
+	gint i = 0, hlen;
+	bool ret = false;
+
+	if (uri->hostlen == 0) {
+		return false;
+	}
+
+	hlen = uri->hostlen;
+	t = rspamd_url_host_unsafe(uri);
+
+	while (i < hlen) {
+		gint prev_i = i;
+		U8_NEXT(hstart, i, hlen, uc);
+
+		if (is_idna_label_dot(uc)) {
+			*t++ = '.';
+			ret = true;
+		}
+		else {
+			if (ret) {
+				/* We have to shift the remaining stuff */
+				while (prev_i < i) {
+					*t++ = *(hstart + prev_i);
+					prev_i++;
+				}
+			}
+			else {
+				t += (i - prev_i);
+			}
+		}
+	}
+
+	if (ret) {
+		rspamd_url_shift(uri, t - hstart, UF_HOST);
+	}
+
+	return ret;
+}
+
+enum uri_errno
+rspamd_url_parse(struct rspamd_url *uri,
+				 gchar *uristring, gsize len,
+				 rspamd_mempool_t *pool,
+				 enum rspamd_url_parse_flags parse_flags)
+{
+	struct http_parser_url u;
+	gchar *p;
+	const gchar *end;
+	guint complen, ret, flags = 0;
+	gsize unquoted_len = 0;
+
+	memset(uri, 0, sizeof(*uri));
+	memset(&u, 0, sizeof(u));
+	uri->count = 1;
+	/* Undefine order */
+	uri->order = -1;
+	uri->part_order = -1;
+
+	if (*uristring == '\0') {
+		return URI_ERRNO_EMPTY;
+	}
+
+	if (len >= G_MAXUINT16 / 2) {
+		flags |= RSPAMD_URL_FLAG_TRUNCATED;
+		len = G_MAXUINT16 / 2;
+	}
+
+	p = uristring;
+	uri->protocol = PROTOCOL_UNKNOWN;
+
+	if (len > sizeof("mailto:") - 1) {
+		/* For mailto: urls we also need to add slashes to make it a valid URL */
+		if (g_ascii_strncasecmp(p, "mailto:", sizeof("mailto:") - 1) == 0) {
+			ret = rspamd_mailto_parse(&u, uristring, len, &end, parse_flags,
+									  &flags);
+		}
+		else if (g_ascii_strncasecmp(p, "tel:", sizeof("tel:") - 1) == 0 ||
+				 g_ascii_strncasecmp(p, "callto:", sizeof("callto:") - 1) == 0) {
+			ret = rspamd_telephone_parse(&u, uristring, len, &end, parse_flags,
+										 &flags);
+			uri->protocol = PROTOCOL_TELEPHONE;
+		}
+		else {
+			ret = rspamd_web_parse(&u, uristring, len, &end, parse_flags,
+								   &flags);
+		}
+	}
+	else {
+		ret = rspamd_web_parse(&u, uristring, len, &end, parse_flags, &flags);
+	}
+
+	if (ret != 0) {
+		return URI_ERRNO_BAD_FORMAT;
+	}
+
+	if (end > uristring && (guint) (end - uristring) != len) {
+		len = end - uristring;
+	}
+
+	uri->raw = p;
+	uri->rawlen = len;
+
+	if (flags & RSPAMD_URL_FLAG_MISSINGSLASHES) {
+		len += 2;
+		uri->string = rspamd_mempool_alloc(pool, len + 1);
+		memcpy(uri->string, p, u.field_data[UF_SCHEMA].len);
+		memcpy(uri->string + u.field_data[UF_SCHEMA].len, "://", 3);
+		rspamd_strlcpy(uri->string + u.field_data[UF_SCHEMA].len + 3,
+					   p + u.field_data[UF_SCHEMA].len + 1,
+					   len - 2 - u.field_data[UF_SCHEMA].len);
+		/* Compensate slashes added */
+		for (int i = UF_SCHEMA + 1; i < UF_MAX; i++) {
+			if (u.field_set & (1 << i)) {
+				u.field_data[i].off += 2;
+			}
+		}
+	}
+	else {
+		uri->string = rspamd_mempool_alloc(pool, len + 1);
+		rspamd_strlcpy(uri->string, p, len + 1);
+	}
+
+	uri->urllen = len;
+	uri->flags = flags;
+
+	for (guint i = 0; i < UF_MAX; i++) {
+		if (u.field_set & (1 << i)) {
+			guint shift = u.field_data[i].off;
+			complen = u.field_data[i].len;
+
+			if (complen >= G_MAXUINT16) {
+				/* Too large component length */
+				return URI_ERRNO_BAD_FORMAT;
+			}
+
+			switch (i) {
+			case UF_SCHEMA:
+				uri->protocollen = u.field_data[i].len;
+				break;
+			case UF_HOST:
+				uri->hostshift = shift;
+				uri->hostlen = complen;
+				break;
+			case UF_PATH:
+				uri->datashift = shift;
+				uri->datalen = complen;
+				break;
+			case UF_QUERY:
+				uri->queryshift = shift;
+				uri->querylen = complen;
+				break;
+			case UF_FRAGMENT:
+				uri->fragmentshift = shift;
+				uri->fragmentlen = complen;
+				break;
+			case UF_USERINFO:
+				uri->usershift = shift;
+				uri->userlen = complen;
+				break;
+			default:
+				break;
+			}
+		}
+	}
+
+	/* Port is 'special' in case of url_parser as it is not a part of UF_* macro logic */
+	if (u.port != 0) {
+		if (!uri->ext) {
+			uri->ext = rspamd_mempool_alloc0_type(pool, struct rspamd_url_ext);
+		}
+		uri->flags |= RSPAMD_URL_FLAG_HAS_PORT;
+		uri->ext->port = u.port;
+	}
+
+	if (!uri->hostlen) {
+		return URI_ERRNO_HOST_MISSING;
+	}
+
+	/* Now decode url symbols */
+	unquoted_len = rspamd_url_decode(uri->string,
+									 uri->string,
+									 uri->protocollen);
+	rspamd_url_shift(uri, unquoted_len, UF_SCHEMA);
+	unquoted_len = rspamd_url_decode(rspamd_url_host_unsafe(uri),
+									 rspamd_url_host_unsafe(uri), uri->hostlen);
+
+	rspamd_url_normalise_propagate_flags(pool, rspamd_url_host_unsafe(uri),
+										 &unquoted_len, uri->flags);
+
+	rspamd_url_shift(uri, unquoted_len, UF_HOST);
+
+	if (rspamd_url_remove_dots(uri)) {
+		uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+	}
+
+	if (uri->protocol & (PROTOCOL_HTTP | PROTOCOL_HTTPS | PROTOCOL_MAILTO | PROTOCOL_FTP | PROTOCOL_FILE)) {
+		/* Ensure that hostname starts with something sane (exclude numeric urls) */
+		const gchar *host = rspamd_url_host_unsafe(uri);
+
+		if (!(is_domain_start(host[0]) || host[0] == ':')) {
+			return URI_ERRNO_BAD_FORMAT;
+		}
+	}
+
+	/* Apply nameprep algorithm */
+	static UStringPrepProfile *nameprep = NULL;
+	UErrorCode uc_err = U_ZERO_ERROR;
+
+	if (nameprep == NULL) {
+		/* Open and cache profile */
+		nameprep = usprep_openByType(USPREP_RFC3491_NAMEPREP, &uc_err);
+
+		g_assert(U_SUCCESS(uc_err));
+	}
+
+	UChar *utf16_hostname, *norm_utf16;
+	gint32 utf16_len, norm_utf16_len, norm_utf8_len;
+	UParseError parse_error;
+
+	utf16_hostname = rspamd_mempool_alloc(pool, uri->hostlen * sizeof(UChar));
+	struct UConverter *utf8_conv = rspamd_get_utf8_converter();
+
+	utf16_len = ucnv_toUChars(utf8_conv, utf16_hostname, uri->hostlen,
+							  rspamd_url_host_unsafe(uri), uri->hostlen, &uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+
+		return URI_ERRNO_BAD_FORMAT;
+	}
+
+	norm_utf16 = rspamd_mempool_alloc(pool, utf16_len * sizeof(UChar));
+	norm_utf16_len = usprep_prepare(nameprep, utf16_hostname, utf16_len,
+									norm_utf16, utf16_len, USPREP_DEFAULT, &parse_error, &uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+
+		return URI_ERRNO_BAD_FORMAT;
+	}
+
+	/* Convert back to utf8, sigh... */
+	norm_utf8_len = ucnv_fromUChars(utf8_conv,
+									rspamd_url_host_unsafe(uri), uri->hostlen,
+									norm_utf16, norm_utf16_len, &uc_err);
+
+	if (!U_SUCCESS(uc_err)) {
+
+		return URI_ERRNO_BAD_FORMAT;
+	}
+
+	/* Final shift of lengths */
+	rspamd_url_shift(uri, norm_utf8_len, UF_HOST);
+
+	/* Process data part */
+	if (uri->datalen) {
+		unquoted_len = rspamd_url_decode(rspamd_url_data_unsafe(uri),
+										 rspamd_url_data_unsafe(uri), uri->datalen);
+
+		rspamd_url_normalise_propagate_flags(pool, rspamd_url_data_unsafe(uri),
+											 &unquoted_len, uri->flags);
+
+		rspamd_url_shift(uri, unquoted_len, UF_PATH);
+		/* We now normalize path */
+		rspamd_normalize_path_inplace(rspamd_url_data_unsafe(uri),
+									  uri->datalen, &unquoted_len);
+		rspamd_url_shift(uri, unquoted_len, UF_PATH);
+	}
+
+	if (uri->querylen) {
+		unquoted_len = rspamd_url_decode(rspamd_url_query_unsafe(uri),
+										 rspamd_url_query_unsafe(uri),
+										 uri->querylen);
+
+		rspamd_url_normalise_propagate_flags(pool, rspamd_url_query_unsafe(uri),
+											 &unquoted_len, uri->flags);
+		rspamd_url_shift(uri, unquoted_len, UF_QUERY);
+	}
+
+	if (uri->fragmentlen) {
+		unquoted_len = rspamd_url_decode(rspamd_url_fragment_unsafe(uri),
+										 rspamd_url_fragment_unsafe(uri),
+										 uri->fragmentlen);
+
+		rspamd_url_normalise_propagate_flags(pool, rspamd_url_fragment_unsafe(uri),
+											 &unquoted_len, uri->flags);
+		rspamd_url_shift(uri, unquoted_len, UF_FRAGMENT);
+	}
+
+	rspamd_str_lc(uri->string, uri->protocollen);
+	unquoted_len = rspamd_str_lc_utf8(rspamd_url_host_unsafe(uri), uri->hostlen);
+	rspamd_url_shift(uri, unquoted_len, UF_HOST);
+
+	if (uri->protocol == PROTOCOL_UNKNOWN) {
+		for (int i = 0; i < G_N_ELEMENTS(rspamd_url_protocols); i++) {
+			if (uri->protocollen == rspamd_url_protocols[i].len) {
+				if (memcmp(uri->string,
+						   rspamd_url_protocols[i].name, uri->protocollen) == 0) {
+					uri->protocol = rspamd_url_protocols[i].proto;
+					break;
+				}
+			}
+		}
+	}
+
+	if (uri->protocol & (PROTOCOL_HTTP | PROTOCOL_HTTPS | PROTOCOL_MAILTO | PROTOCOL_FTP | PROTOCOL_FILE)) {
+		/* Find TLD part */
+		if (url_scanner->search_trie_full) {
+			rspamd_multipattern_lookup(url_scanner->search_trie_full,
+									   rspamd_url_host_unsafe(uri), uri->hostlen,
+									   rspamd_tld_trie_callback, uri, NULL);
+		}
+
+		if (uri->tldlen == 0) {
+			/*
+			 * If we have not detected eSLD, but there are no dots in the hostname,
+			 * then we should treat the whole hostname as eSLD - a rule of thumb
+			 *
+			 * We also check that a hostname ends with a permitted character, and all characters are forming
+			 * DNS label. We also need to check for a numeric IP within this check.
+			 */
+			const char *dot_pos = memchr(rspamd_url_host_unsafe(uri), '.', uri->hostlen);
+			bool is_whole_hostname_tld = false;
+
+			if (uri->hostlen > 0 && (dot_pos == NULL || dot_pos == rspamd_url_host_unsafe(uri) + uri->hostlen - 1)) {
+				bool all_chars_domain = true;
+
+				for (int i = 0; i < uri->hostlen; i++) {
+					if (!is_domain(rspamd_url_host_unsafe(uri)[i])) {
+						all_chars_domain = false;
+						break;
+					}
+				}
+
+				char last_c = rspamd_url_host_unsafe(uri)[uri->hostlen - 1];
+
+				if (all_chars_domain) {
+					/* Also check the last character to be either a dot or alphanumeric character */
+					if (last_c != '.' && !g_ascii_isalnum(last_c)) {
+						all_chars_domain = false;
+					}
+				}
+
+				if (all_chars_domain) {
+					/* Additionally check for a numeric IP as we can have some number here... */
+					rspamd_url_maybe_regenerate_from_ip(uri, pool);
+
+					if (last_c == '.' && uri->hostlen > 1) {
+						/* Skip the last dot */
+						uri->tldlen = uri->hostlen - 1;
+					}
+					else {
+						uri->tldlen = uri->hostlen;
+					}
+
+					uri->tldshift = uri->hostshift;
+					is_whole_hostname_tld = true;
+				}
+			}
+
+			if (!is_whole_hostname_tld) {
+				if (uri->protocol != PROTOCOL_MAILTO) {
+					if (url_scanner->has_tld_file && !(parse_flags & RSPAMD_URL_PARSE_HREF)) {
+						/* Ignore URL's without TLD if it is not a numeric URL */
+						if (!rspamd_url_maybe_regenerate_from_ip(uri, pool)) {
+							return URI_ERRNO_TLD_MISSING;
+						}
+					}
+					else {
+						if (!rspamd_url_maybe_regenerate_from_ip(uri, pool)) {
+							/* Assume tld equal to host */
+							uri->tldshift = uri->hostshift;
+							uri->tldlen = uri->hostlen;
+						}
+						else if (uri->flags & RSPAMD_URL_FLAG_SCHEMALESS) {
+							/* Ignore urls with both no schema and no tld */
+							return URI_ERRNO_TLD_MISSING;
+						}
+
+						uri->flags |= RSPAMD_URL_FLAG_NO_TLD;
+					}
+				}
+				else {
+					/* Ignore IP like domains for mailto, as it is really never supported */
+					return URI_ERRNO_TLD_MISSING;
+				}
+			}
+		}
+
+		/* Replace stupid '\' with '/' after schema */
+		if (uri->protocol & (PROTOCOL_HTTP | PROTOCOL_HTTPS | PROTOCOL_FTP) &&
+			uri->protocollen > 0 && uri->urllen > uri->protocollen + 2) {
+
+			gchar *pos = &uri->string[uri->protocollen],
+				  *host_start = rspamd_url_host_unsafe(uri);
+
+			while (pos < host_start) {
+				if (*pos == '\\') {
+					*pos = '/';
+					uri->flags |= RSPAMD_URL_FLAG_OBSCURED;
+				}
+				pos++;
+			}
+		}
+	}
+	else if (uri->protocol & PROTOCOL_TELEPHONE) {
+		/* We need to normalise phone number: remove all spaces and braces */
+		rspamd_telephone_normalise_inplace(uri);
+
+		if (rspamd_url_host_unsafe(uri)[0] == '+') {
+			uri->tldshift = uri->hostshift + 1;
+			uri->tldlen = uri->hostlen - 1;
+		}
+		else {
+			uri->tldshift = uri->hostshift;
+			uri->tldlen = uri->hostlen;
+		}
+	}
+
+	if (uri->protocol == PROTOCOL_UNKNOWN) {
+		if (!(parse_flags & RSPAMD_URL_PARSE_HREF)) {
+			return URI_ERRNO_INVALID_PROTOCOL;
+		}
+		else {
+			/* Hack, hack, hack */
+			uri->protocol = PROTOCOL_UNKNOWN;
+		}
+	}
+
+	return URI_ERRNO_OK;
+}
+
+struct tld_trie_cbdata {
+	const gchar *begin;
+	gsize len;
+	rspamd_ftok_t *out;
+};
+
+static gint
+rspamd_tld_trie_find_callback(struct rspamd_multipattern *mp,
+							  guint strnum,
+							  gint match_start,
+							  gint match_pos,
+							  const gchar *text,
+							  gsize len,
+							  void *context)
+{
+	struct url_matcher *matcher;
+	const gchar *start, *pos, *p;
+	struct tld_trie_cbdata *cbdata = context;
+	gint ndots = 1;
+
+	matcher = &g_array_index(url_scanner->matchers_full, struct url_matcher,
+							 strnum);
+
+	if (matcher->flags & URL_MATCHER_FLAG_STAR_MATCH) {
+		/* Skip one more tld component */
+		ndots = 2;
+	}
+
+	pos = text + match_start;
+	p = pos - 1;
+	start = text;
+
+	if (*pos != '.' || match_pos != (gint) cbdata->len) {
+		/* Something weird has been found */
+		if (match_pos != (gint) cbdata->len - 1) {
+			/* Search more */
+			return 0;
+		}
+	}
+
+	/* Now we need to find top level domain */
+	pos = start;
+
+	while (p >= start && ndots > 0) {
+		if (*p == '.') {
+			ndots--;
+			pos = p + 1;
+		}
+		else {
+			pos = p;
+		}
+
+		p--;
+	}
+
+	if (ndots == 0 || p == start - 1) {
+		if (cbdata->begin + cbdata->len - pos > cbdata->out->len) {
+			cbdata->out->begin = pos;
+			cbdata->out->len = cbdata->begin + cbdata->len - pos;
+		}
+	}
+
+	return 0;
+}
+
+gboolean
+rspamd_url_find_tld(const gchar *in, gsize inlen, rspamd_ftok_t *out)
+{
+	struct tld_trie_cbdata cbdata;
+
+	g_assert(in != NULL);
+	g_assert(out != NULL);
+	g_assert(url_scanner != NULL);
+
+	cbdata.begin = in;
+	cbdata.len = inlen;
+	cbdata.out = out;
+	out->len = 0;
+
+	if (url_scanner->search_trie_full) {
+		rspamd_multipattern_lookup(url_scanner->search_trie_full, in, inlen,
+								   rspamd_tld_trie_find_callback, &cbdata, NULL);
+	}
+
+	if (out->len > 0) {
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static const gchar url_braces[] = {
+	'(', ')',
+	'{', '}',
+	'[', ']',
+	'<', '>',
+	'|', '|',
+	'\'', '\''};
+
+
+static gboolean
+url_file_start(struct url_callback_data *cb,
+			   const gchar *pos,
+			   url_match_t *match)
+{
+	match->m_begin = pos;
+
+	if (pos > cb->begin) {
+		match->st = *(pos - 1);
+	}
+	else {
+		match->st = '\0';
+	}
+
+	return TRUE;
+}
+
+static gboolean
+url_file_end(struct url_callback_data *cb,
+			 const gchar *pos,
+			 url_match_t *match)
+{
+	const gchar *p;
+	gchar stop;
+	guint i;
+
+	p = pos + strlen(match->pattern);
+	stop = *p;
+	if (*p == '/') {
+		p++;
+	}
+
+	for (i = 0; i < G_N_ELEMENTS(url_braces) / 2; i += 2) {
+		if (*p == url_braces[i]) {
+			stop = url_braces[i + 1];
+			break;
+		}
+	}
+
+	while (p < cb->end && *p != stop && is_urlsafe(*p)) {
+		p++;
+	}
+
+	if (p == cb->begin) {
+		return FALSE;
+	}
+	match->m_len = p - match->m_begin;
+
+	return TRUE;
+}
+
+static gboolean
+url_tld_start(struct url_callback_data *cb,
+			  const gchar *pos,
+			  url_match_t *match)
+{
+	const gchar *p = pos;
+	guint processed = 0;
+	static const guint max_shift = 253 + sizeof("https://");
+
+	/* Try to find the start of the url by finding any non-urlsafe character or whitespace/punctuation */
+	while (p >= cb->begin) {
+		if (!is_domain(*p) || g_ascii_isspace(*p) || is_url_start(*p) ||
+			p == match->prev_newline_pos) {
+			if (!is_url_start(*p) && !g_ascii_isspace(*p) &&
+				p != match->prev_newline_pos) {
+				return FALSE;
+			}
+
+			if (p != match->prev_newline_pos) {
+				match->st = *p;
+
+				p++;
+			}
+			else {
+				match->st = '\n';
+			}
+
+			if (!g_ascii_isalnum(*p)) {
+				/* Urls cannot start with strange symbols */
+				return FALSE;
+			}
+
+			match->m_begin = p;
+			return TRUE;
+		}
+		else if (p == cb->begin && p != pos) {
+			match->st = '\0';
+			match->m_begin = p;
+
+			return TRUE;
+		}
+		else if (*p == '.') {
+			if (p == cb->begin) {
+				/* Urls cannot start with a dot */
+				return FALSE;
+			}
+			if (!g_ascii_isalnum(p[1])) {
+				/* Wrong we have an invalid character after dot */
+				return FALSE;
+			}
+		}
+		else if (*p == '/') {
+			/* Urls cannot contain '/' in their body */
+			return FALSE;
+		}
+
+		p--;
+		processed++;
+
+		if (processed > max_shift) {
+			/* Too long */
+			return FALSE;
+		}
+	}
+
+	return FALSE;
+}
+
+static gboolean
+url_tld_end(struct url_callback_data *cb,
+			const gchar *pos,
+			url_match_t *match)
+{
+	const gchar *p;
+	gboolean ret = FALSE;
+
+	p = pos + match->m_len;
+
+	if (p == cb->end) {
+		match->m_len = p - match->m_begin;
+		return TRUE;
+	}
+	else if (*p == '/' || *p == ':' || is_url_end(*p) || is_lwsp(*p) ||
+			 (match->st != '<' && p == match->newline_pos)) {
+		/* Parse arguments, ports by normal way by url default function */
+		p = match->m_begin;
+		/* Check common prefix */
+		if (g_ascii_strncasecmp(p, "http://", sizeof("http://") - 1) == 0) {
+			ret = url_web_end(cb,
+							  match->m_begin + sizeof("http://") - 1,
+							  match);
+		}
+		else {
+			ret = url_web_end(cb, match->m_begin, match);
+		}
+	}
+	else if (*p == '.') {
+		p++;
+		if (p < cb->end) {
+			if (g_ascii_isspace(*p) || *p == '/' ||
+				*p == '?' || *p == ':') {
+				ret = url_web_end(cb, match->m_begin, match);
+			}
+		}
+	}
+
+	if (ret) {
+		/* Check sanity of match found */
+		if (match->m_begin + match->m_len <= pos) {
+			return FALSE;
+		}
+	}
+
+	return ret;
+}
+
+static gboolean
+url_web_start(struct url_callback_data *cb,
+			  const gchar *pos,
+			  url_match_t *match)
+{
+	/* Check what we have found */
+	if (pos > cb->begin) {
+		if (g_ascii_strncasecmp(pos, "www", 3) == 0) {
+
+			if (!(is_url_start(*(pos - 1)) ||
+				  g_ascii_isspace(*(pos - 1)) ||
+				  pos - 1 == match->prev_newline_pos ||
+				  (*(pos - 1) & 0x80))) { /* Chinese trick */
+				return FALSE;
+			}
+		}
+		else {
+			guchar prev = *(pos - 1);
+
+			if (g_ascii_isalnum(prev)) {
+				/* Part of another url */
+				return FALSE;
+			}
+		}
+	}
+
+	if (*pos == '.') {
+		/* Urls cannot start with . */
+		return FALSE;
+	}
+
+	if (pos > cb->begin) {
+		match->st = *(pos - 1);
+	}
+	else {
+		match->st = '\0';
+	}
+
+	match->m_begin = pos;
+
+	return TRUE;
+}
+
+static gboolean
+url_web_end(struct url_callback_data *cb,
+			const gchar *pos,
+			url_match_t *match)
+{
+	const gchar *last = NULL;
+	gint len = cb->end - pos;
+	guint flags = 0;
+
+	if (match->newline_pos && match->st != '<') {
+		/* We should also limit our match end to the newline */
+		len = MIN(len, match->newline_pos - pos);
+	}
+
+	if (rspamd_web_parse(NULL, pos, len, &last,
+						 RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
+		return FALSE;
+	}
+
+	if (last < cb->end && (*last == '>' && last != match->newline_pos)) {
+		/* We need to ensure that url also starts with '>' */
+		if (match->st != '<') {
+			if (last + 1 < cb->end) {
+				if (g_ascii_isspace(last[1])) {
+					return FALSE;
+				}
+			}
+			else {
+				return FALSE;
+			}
+		}
+	}
+
+	match->m_len = (last - pos);
+	cb->fin = last + 1;
+
+	return TRUE;
+}
+
+
+static gboolean
+url_email_start(struct url_callback_data *cb,
+				const gchar *pos,
+				url_match_t *match)
+{
+	if (!match->prefix || match->prefix[0] == '\0') {
+		/* We have mailto:// at the beginning */
+		match->m_begin = pos;
+
+		if (pos >= cb->begin + 1) {
+			match->st = *(pos - 1);
+		}
+		else {
+			match->st = '\0';
+		}
+	}
+	else {
+		/* Just '@' */
+
+		/* Check if this match is a part of the previous mailto: email */
+		if (cb->last_at != NULL && cb->last_at == pos) {
+			cb->last_at = NULL;
+			return FALSE;
+		}
+		else if (pos == cb->begin) {
+			/* Just @ at the start of input */
+			return FALSE;
+		}
+
+		match->st = '\0';
+	}
+
+	return TRUE;
+}
+
+static gboolean
+url_email_end(struct url_callback_data *cb,
+			  const gchar *pos,
+			  url_match_t *match)
+{
+	const gchar *last = NULL;
+	struct http_parser_url u;
+	gint len = cb->end - pos;
+	guint flags = 0;
+
+	if (match->newline_pos && match->st != '<') {
+		/* We should also limit our match end to the newline */
+		len = MIN(len, match->newline_pos - pos);
+	}
+
+	if (!match->prefix || match->prefix[0] == '\0') {
+		/* We have mailto:// at the beginning */
+		if (rspamd_mailto_parse(&u, pos, len, &last,
+								RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
+			return FALSE;
+		}
+
+		if (!(u.field_set & (1 << UF_USERINFO))) {
+			return FALSE;
+		}
+
+		cb->last_at = match->m_begin + u.field_data[UF_USERINFO].off +
+					  u.field_data[UF_USERINFO].len;
+
+		g_assert(*cb->last_at == '@');
+		match->m_len = (last - pos);
+
+		return TRUE;
+	}
+	else {
+		const gchar *c, *p;
+		/*
+		 * Here we have just '@', so we need to find both start and end of the
+		 * pattern
+		 */
+		g_assert(*pos == '@');
+
+		if (pos >= cb->end - 2 || pos < cb->begin + 1) {
+			/* Boundary violation */
+			return FALSE;
+		}
+
+		/* Check the next character after `@` */
+		if (!g_ascii_isalnum(pos[1]) || !g_ascii_isalnum(*(pos - 1))) {
+			return FALSE;
+		}
+
+
+		c = pos - 1;
+		while (c > cb->begin) {
+			if (!is_mailsafe(*c)) {
+				break;
+			}
+			if (c == match->prev_newline_pos) {
+				break;
+			}
+
+			c--;
+		}
+		/* Rewind to the first alphanumeric character */
+		while (c < pos && !g_ascii_isalnum(*c)) {
+			c++;
+		}
+
+		/* Find the end of email */
+		p = pos + 1;
+		while (p < cb->end && is_domain(*p)) {
+			if (p == match->newline_pos) {
+				break;
+			}
+
+			p++;
+		}
+
+		/* Rewind it again to avoid bad emails to be detected */
+		while (p > pos && p < cb->end && !g_ascii_isalnum(*p)) {
+			p--;
+		}
+
+		if (p < cb->end && g_ascii_isalnum(*p) &&
+			(match->newline_pos == NULL || p < match->newline_pos)) {
+			p++;
+		}
+
+		if (p > c) {
+			match->m_begin = c;
+			match->m_len = p - c;
+			return TRUE;
+		}
+	}
+
+	return FALSE;
+}
+
+static gboolean
+url_tel_start(struct url_callback_data *cb,
+			  const gchar *pos,
+			  url_match_t *match)
+{
+	match->m_begin = pos;
+
+	if (pos >= cb->begin + 1) {
+		match->st = *(pos - 1);
+	}
+	else {
+		match->st = '\0';
+	}
+
+	return TRUE;
+}
+
+static gboolean
+url_tel_end(struct url_callback_data *cb,
+			const gchar *pos,
+			url_match_t *match)
+{
+	const gchar *last = NULL;
+	struct http_parser_url u;
+	gint len = cb->end - pos;
+	guint flags = 0;
+
+	if (match->newline_pos && match->st != '<') {
+		/* We should also limit our match end to the newline */
+		len = MIN(len, match->newline_pos - pos);
+	}
+
+	if (rspamd_telephone_parse(&u, pos, len, &last,
+							   RSPAMD_URL_PARSE_CHECK, &flags) != 0) {
+		return FALSE;
+	}
+
+	if (!(u.field_set & (1 << UF_HOST))) {
+		return FALSE;
+	}
+
+	match->m_len = (last - pos);
+
+	return TRUE;
+}
+
+
+static gboolean
+rspamd_url_trie_is_match(struct url_matcher *matcher, const gchar *pos,
+						 const gchar *end, const gchar *newline_pos)
+{
+	if (matcher->flags & URL_MATCHER_FLAG_TLD_MATCH) {
+		/* Immediately check pos for valid chars */
+		if (pos < end) {
+			if (pos != newline_pos && !g_ascii_isspace(*pos) && *pos != '/' && *pos != '?' &&
+				*pos != ':' && !is_url_end(*pos)) {
+				if (*pos == '.') {
+					/* We allow . at the end of the domain however */
+					pos++;
+					if (pos < end) {
+						if (!g_ascii_isspace(*pos) && *pos != '/' &&
+							*pos != '?' && *pos != ':' && !is_url_end(*pos)) {
+							return FALSE;
+						}
+					}
+				}
+				else {
+					return FALSE;
+				}
+			}
+		}
+	}
+
+	return TRUE;
+}
+
+static gint
+rspamd_url_trie_callback(struct rspamd_multipattern *mp,
+						 guint strnum,
+						 gint match_start,
+						 gint match_pos,
+						 const gchar *text,
+						 gsize len,
+						 void *context)
+{
+	struct url_matcher *matcher;
+	url_match_t m;
+	const gchar *pos, *newline_pos = NULL;
+	struct url_callback_data *cb = context;
+
+	pos = text + match_pos;
+
+	if (cb->fin > pos) {
+		/* Already seen */
+		return 0;
+	}
+
+	matcher = &g_array_index(cb->matchers, struct url_matcher,
+							 strnum);
+
+	if ((matcher->flags & URL_MATCHER_FLAG_NOHTML) && cb->how == RSPAMD_URL_FIND_STRICT) {
+		/* Do not try to match non-html like urls in html texts */
+		return 0;
+	}
+
+	memset(&m, 0, sizeof(m));
+	m.m_begin = text + match_start;
+	m.m_len = match_pos - match_start;
+
+	if (cb->newlines && cb->newlines->len > 0) {
+		newline_pos = g_ptr_array_index(cb->newlines, cb->newline_idx);
+
+		while (pos > newline_pos && cb->newline_idx < cb->newlines->len) {
+			cb->newline_idx++;
+			newline_pos = g_ptr_array_index(cb->newlines, cb->newline_idx);
+		}
+
+		if (pos > newline_pos) {
+			newline_pos = NULL;
+		}
+
+		if (cb->newline_idx > 0) {
+			m.prev_newline_pos = g_ptr_array_index(cb->newlines,
+												   cb->newline_idx - 1);
+		}
+	}
+
+	if (!rspamd_url_trie_is_match(matcher, pos, cb->end, newline_pos)) {
+		return 0;
+	}
+
+	m.pattern = matcher->pattern;
+	m.prefix = matcher->prefix;
+	m.add_prefix = FALSE;
+	m.newline_pos = newline_pos;
+	pos = cb->begin + match_start;
+
+	if (matcher->start(cb, pos, &m) &&
+		matcher->end(cb, pos, &m)) {
+		if (m.add_prefix || matcher->prefix[0] != '\0') {
+			cb->len = m.m_len + strlen(matcher->prefix);
+			cb->url_str = rspamd_mempool_alloc(cb->pool, cb->len + 1);
+			cb->len = rspamd_snprintf(cb->url_str,
+									  cb->len + 1,
+									  "%s%*s",
+									  m.prefix,
+									  (gint) m.m_len,
+									  m.m_begin);
+			cb->prefix_added = TRUE;
+		}
+		else {
+			cb->url_str = rspamd_mempool_alloc(cb->pool, m.m_len + 1);
+			rspamd_strlcpy(cb->url_str, m.m_begin, m.m_len + 1);
+		}
+
+		cb->start = m.m_begin;
+
+		if (pos > cb->fin) {
+			cb->fin = pos;
+		}
+
+		return 1;
+	}
+	else {
+		cb->url_str = NULL;
+	}
+
+	/* Continue search */
+	return 0;
+}
+
+gboolean
+rspamd_url_find(rspamd_mempool_t *pool,
+				const gchar *begin, gsize len,
+				gchar **url_str,
+				enum rspamd_url_find_type how,
+				goffset *url_pos,
+				gboolean *prefix_added)
+{
+	struct url_callback_data cb;
+	gint ret;
+
+	memset(&cb, 0, sizeof(cb));
+	cb.begin = begin;
+	cb.end = begin + len;
+	cb.how = how;
+	cb.pool = pool;
+
+	if (how == RSPAMD_URL_FIND_ALL) {
+		if (url_scanner->search_trie_full) {
+			cb.matchers = url_scanner->matchers_full;
+			ret = rspamd_multipattern_lookup(url_scanner->search_trie_full,
+											 begin, len,
+											 rspamd_url_trie_callback, &cb, NULL);
+		}
+		else {
+			cb.matchers = url_scanner->matchers_strict;
+			ret = rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+											 begin, len,
+											 rspamd_url_trie_callback, &cb, NULL);
+		}
+	}
+	else {
+		cb.matchers = url_scanner->matchers_strict;
+		ret = rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+										 begin, len,
+										 rspamd_url_trie_callback, &cb, NULL);
+	}
+
+	if (ret) {
+		if (url_str) {
+			*url_str = cb.url_str;
+		}
+
+		if (url_pos) {
+			*url_pos = cb.start - begin;
+		}
+
+		if (prefix_added) {
+			*prefix_added = cb.prefix_added;
+		}
+
+		return TRUE;
+	}
+
+	return FALSE;
+}
+
+static gint
+rspamd_url_trie_generic_callback_common(struct rspamd_multipattern *mp,
+										guint strnum,
+										gint match_start,
+										gint match_pos,
+										const gchar *text,
+										gsize len,
+										void *context,
+										gboolean multiple)
+{
+	struct rspamd_url *url;
+	struct url_matcher *matcher;
+	url_match_t m;
+	const gchar *pos, *newline_pos = NULL;
+	struct url_callback_data *cb = context;
+	gint rc;
+	rspamd_mempool_t *pool;
+
+	pos = text + match_pos;
+
+	if (cb->fin > pos) {
+		/* Already seen */
+		return 0;
+	}
+
+	matcher = &g_array_index(cb->matchers, struct url_matcher,
+							 strnum);
+	pool = cb->pool;
+
+	if ((matcher->flags & URL_MATCHER_FLAG_NOHTML) && cb->how == RSPAMD_URL_FIND_STRICT) {
+		/* Do not try to match non-html like urls in html texts, continue matching */
+		return 0;
+	}
+
+	memset(&m, 0, sizeof(m));
+
+
+	/* Find the next newline after our pos */
+	if (cb->newlines && cb->newlines->len > 0) {
+		newline_pos = g_ptr_array_index(cb->newlines, cb->newline_idx);
+
+		while (pos > newline_pos && cb->newline_idx < cb->newlines->len - 1) {
+			cb->newline_idx++;
+			newline_pos = g_ptr_array_index(cb->newlines, cb->newline_idx);
+		}
+
+		if (pos > newline_pos) {
+			newline_pos = NULL;
+		}
+		if (cb->newline_idx > 0) {
+			m.prev_newline_pos = g_ptr_array_index(cb->newlines,
+												   cb->newline_idx - 1);
+		}
+	}
+
+	if (!rspamd_url_trie_is_match(matcher, pos, text + len, newline_pos)) {
+		/* Mismatch, continue */
+		return 0;
+	}
+
+	pos = cb->begin + match_start;
+	m.pattern = matcher->pattern;
+	m.prefix = matcher->prefix;
+	m.add_prefix = FALSE;
+	m.m_begin = text + match_start;
+	m.m_len = match_pos - match_start;
+	m.newline_pos = newline_pos;
+
+	if (matcher->start(cb, pos, &m) &&
+		matcher->end(cb, pos, &m)) {
+		if (m.add_prefix || matcher->prefix[0] != '\0') {
+			cb->len = m.m_len + strlen(matcher->prefix);
+			cb->url_str = rspamd_mempool_alloc(cb->pool, cb->len + 1);
+			cb->len = rspamd_snprintf(cb->url_str,
+									  cb->len + 1,
+									  "%s%*s",
+									  m.prefix,
+									  (gint) m.m_len,
+									  m.m_begin);
+			cb->prefix_added = TRUE;
+		}
+		else {
+			cb->url_str = rspamd_mempool_alloc(cb->pool, m.m_len + 1);
+			cb->len = rspamd_strlcpy(cb->url_str, m.m_begin, m.m_len + 1);
+		}
+
+		cb->start = m.m_begin;
+
+		if (pos > cb->fin) {
+			cb->fin = pos;
+		}
+
+		url = rspamd_mempool_alloc0(pool, sizeof(struct rspamd_url));
+		g_strstrip(cb->url_str);
+		rc = rspamd_url_parse(url, cb->url_str,
+							  strlen(cb->url_str), pool,
+							  RSPAMD_URL_PARSE_TEXT);
+
+		if (rc == URI_ERRNO_OK && url->hostlen > 0) {
+			if (cb->prefix_added) {
+				url->flags |= RSPAMD_URL_FLAG_SCHEMALESS;
+				cb->prefix_added = FALSE;
+			}
+
+			if (cb->func) {
+				if (!cb->func(url, cb->start - text, (m.m_begin + m.m_len) - text,
+							  cb->funcd)) {
+					/* We need to stop here in any case! */
+					return -1;
+				}
+			}
+		}
+		else if (rc != URI_ERRNO_OK) {
+			msg_debug_pool_check("extract of url '%s' failed: %s",
+								 cb->url_str,
+								 rspamd_url_strerror(rc));
+		}
+	}
+	else {
+		cb->url_str = NULL;
+		/* Continue search if no pattern has been found */
+		return 0;
+	}
+
+	/* Continue search if required (return 0 means continue) */
+	return !multiple;
+}
+
+static gint
+rspamd_url_trie_generic_callback_multiple(struct rspamd_multipattern *mp,
+										  guint strnum,
+										  gint match_start,
+										  gint match_pos,
+										  const gchar *text,
+										  gsize len,
+										  void *context)
+{
+	return rspamd_url_trie_generic_callback_common(mp, strnum, match_start,
+												   match_pos, text, len, context, TRUE);
+}
+
+static gint
+rspamd_url_trie_generic_callback_single(struct rspamd_multipattern *mp,
+										guint strnum,
+										gint match_start,
+										gint match_pos,
+										const gchar *text,
+										gsize len,
+										void *context)
+{
+	return rspamd_url_trie_generic_callback_common(mp, strnum, match_start,
+												   match_pos, text, len, context, FALSE);
+}
+
+struct rspamd_url_mimepart_cbdata {
+	struct rspamd_task *task;
+	struct rspamd_mime_text_part *part;
+	gsize url_len;
+	uint16_t *cur_url_order; /* Global ordering */
+	uint16_t cur_part_order; /* Per part ordering */
+};
+
+static gboolean
+rspamd_url_query_callback(struct rspamd_url *url, gsize start_offset,
+						  gsize end_offset, gpointer ud)
+{
+	struct rspamd_url_mimepart_cbdata *cbd =
+		(struct rspamd_url_mimepart_cbdata *) ud;
+	struct rspamd_task *task;
+
+	task = cbd->task;
+
+	if (url->protocol == PROTOCOL_MAILTO) {
+		if (url->userlen == 0) {
+			return FALSE;
+		}
+	}
+	/* Also check max urls */
+	if (cbd->task->cfg && cbd->task->cfg->max_urls > 0) {
+		if (kh_size(MESSAGE_FIELD(task, urls)) > cbd->task->cfg->max_urls) {
+			msg_err_task("part has too many URLs, we cannot process more: "
+						 "%d urls extracted ",
+						 (guint) kh_size(MESSAGE_FIELD(task, urls)));
+
+			return FALSE;
+		}
+	}
+
+	url->flags |= RSPAMD_URL_FLAG_QUERY;
+
+
+	if (rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url, false)) {
+		if (cbd->part && cbd->part->mime_part->urls) {
+			g_ptr_array_add(cbd->part->mime_part->urls, url);
+		}
+
+		url->part_order = cbd->cur_part_order++;
+
+		if (cbd->cur_url_order) {
+			url->order = (*cbd->cur_url_order)++;
+		}
+	}
+
+	return TRUE;
+}
+
+static gboolean
+rspamd_url_text_part_callback(struct rspamd_url *url, gsize start_offset,
+							  gsize end_offset, gpointer ud)
+{
+	struct rspamd_url_mimepart_cbdata *cbd =
+		(struct rspamd_url_mimepart_cbdata *) ud;
+	struct rspamd_process_exception *ex;
+	struct rspamd_task *task;
+
+	task = cbd->task;
+	ex = rspamd_mempool_alloc0(task->task_pool, sizeof(struct rspamd_process_exception));
+
+	ex->pos = start_offset;
+	ex->len = end_offset - start_offset;
+	ex->type = RSPAMD_EXCEPTION_URL;
+	ex->ptr = url;
+
+	cbd->url_len += ex->len;
+
+	if (cbd->part->utf_stripped_content &&
+		cbd->url_len > cbd->part->utf_stripped_content->len * 10) {
+		/* Absurd case, stop here now */
+		msg_err_task("part has too many URLs, we cannot process more: %z url len; "
+					 "%d stripped content length",
+					 cbd->url_len, cbd->part->utf_stripped_content->len);
+
+		return FALSE;
+	}
+
+	if (url->protocol == PROTOCOL_MAILTO) {
+		if (url->userlen == 0) {
+			return FALSE;
+		}
+	}
+	/* Also check max urls */
+	if (cbd->task->cfg && cbd->task->cfg->max_urls > 0) {
+		if (kh_size(MESSAGE_FIELD(task, urls)) > cbd->task->cfg->max_urls) {
+			msg_err_task("part has too many URLs, we cannot process more: "
+						 "%d urls extracted ",
+						 (guint) kh_size(MESSAGE_FIELD(task, urls)));
+
+			return FALSE;
+		}
+	}
+
+	url->flags |= RSPAMD_URL_FLAG_FROM_TEXT;
+
+	if (rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url, false) &&
+		cbd->part->mime_part->urls) {
+		url->part_order = cbd->cur_part_order++;
+
+		if (cbd->cur_url_order) {
+			url->order = (*cbd->cur_url_order)++;
+		}
+		g_ptr_array_add(cbd->part->mime_part->urls, url);
+	}
+
+	cbd->part->exceptions = g_list_prepend(
+		cbd->part->exceptions,
+		ex);
+
+	/* We also search the query for additional url inside */
+	if (url->querylen > 0) {
+		rspamd_url_find_multiple(task->task_pool,
+								 rspamd_url_query_unsafe(url), url->querylen,
+								 RSPAMD_URL_FIND_ALL, NULL,
+								 rspamd_url_query_callback, cbd);
+	}
+
+	return TRUE;
+}
+
+void rspamd_url_text_extract(rspamd_mempool_t *pool,
+							 struct rspamd_task *task,
+							 struct rspamd_mime_text_part *part,
+							 uint16_t *cur_url_order,
+							 enum rspamd_url_find_type how)
+{
+	struct rspamd_url_mimepart_cbdata mcbd;
+
+	if (part->utf_stripped_content == NULL || part->utf_stripped_content->len == 0) {
+		msg_warn_task("got empty text part");
+		return;
+	}
+
+	mcbd.task = task;
+	mcbd.part = part;
+	mcbd.url_len = 0;
+	mcbd.cur_url_order = cur_url_order;
+	mcbd.cur_part_order = 0;
+
+	rspamd_url_find_multiple(task->task_pool, part->utf_stripped_content->data,
+							 part->utf_stripped_content->len, how, part->newlines,
+							 rspamd_url_text_part_callback, &mcbd);
+}
+
+void rspamd_url_find_multiple(rspamd_mempool_t *pool,
+							  const gchar *in,
+							  gsize inlen,
+							  enum rspamd_url_find_type how,
+							  GPtrArray *nlines,
+							  url_insert_function func,
+							  gpointer ud)
+{
+	struct url_callback_data cb;
+
+	g_assert(in != NULL);
+
+	if (inlen == 0) {
+		inlen = strlen(in);
+	}
+
+	memset(&cb, 0, sizeof(cb));
+	cb.begin = in;
+	cb.end = in + inlen;
+	cb.how = how;
+	cb.pool = pool;
+
+	cb.funcd = ud;
+	cb.func = func;
+	cb.newlines = nlines;
+
+	if (how == RSPAMD_URL_FIND_ALL) {
+		if (url_scanner->search_trie_full) {
+			cb.matchers = url_scanner->matchers_full;
+			rspamd_multipattern_lookup(url_scanner->search_trie_full,
+									   in, inlen,
+									   rspamd_url_trie_generic_callback_multiple, &cb, NULL);
+		}
+		else {
+			cb.matchers = url_scanner->matchers_strict;
+			rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+									   in, inlen,
+									   rspamd_url_trie_generic_callback_multiple, &cb, NULL);
+		}
+	}
+	else {
+		cb.matchers = url_scanner->matchers_strict;
+		rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+								   in, inlen,
+								   rspamd_url_trie_generic_callback_multiple, &cb, NULL);
+	}
+}
+
+void rspamd_url_find_single(rspamd_mempool_t *pool,
+							const gchar *in,
+							gsize inlen,
+							enum rspamd_url_find_type how,
+							url_insert_function func,
+							gpointer ud)
+{
+	struct url_callback_data cb;
+
+	g_assert(in != NULL);
+
+	if (inlen == 0) {
+		inlen = strlen(in);
+	}
+
+	/*
+	 * We might have a situation when we need to parse URLs on config file
+	 * parsing, but there is no valid url_scanner loaded. Hence, we just load
+	 * some defaults and it should be fine...
+	 */
+	if (url_scanner == NULL) {
+		rspamd_url_init(NULL);
+	}
+
+	memset(&cb, 0, sizeof(cb));
+	cb.begin = in;
+	cb.end = in + inlen;
+	cb.how = how;
+	cb.pool = pool;
+
+	cb.funcd = ud;
+	cb.func = func;
+
+	if (how == RSPAMD_URL_FIND_ALL) {
+		if (url_scanner->search_trie_full) {
+			cb.matchers = url_scanner->matchers_full;
+			rspamd_multipattern_lookup(url_scanner->search_trie_full,
+									   in, inlen,
+									   rspamd_url_trie_generic_callback_single, &cb, NULL);
+		}
+		else {
+			cb.matchers = url_scanner->matchers_strict;
+			rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+									   in, inlen,
+									   rspamd_url_trie_generic_callback_single, &cb, NULL);
+		}
+	}
+	else {
+		cb.matchers = url_scanner->matchers_strict;
+		rspamd_multipattern_lookup(url_scanner->search_trie_strict,
+								   in, inlen,
+								   rspamd_url_trie_generic_callback_single, &cb, NULL);
+	}
+}
+
+
+gboolean
+rspamd_url_task_subject_callback(struct rspamd_url *url, gsize start_offset,
+								 gsize end_offset, gpointer ud)
+{
+	struct rspamd_task *task = ud;
+	gchar *url_str = NULL;
+	struct rspamd_url *query_url;
+	gint rc;
+	gboolean prefix_added;
+
+	/* It is just a displayed URL, we should not check it for certain things */
+	url->flags |= RSPAMD_URL_FLAG_HTML_DISPLAYED | RSPAMD_URL_FLAG_SUBJECT;
+
+	if (url->protocol == PROTOCOL_MAILTO) {
+		if (url->userlen == 0) {
+			return FALSE;
+		}
+	}
+
+	rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls), url, false);
+
+	/* We also search the query for additional url inside */
+	if (url->querylen > 0) {
+		if (rspamd_url_find(task->task_pool, rspamd_url_query_unsafe(url), url->querylen,
+							&url_str, RSPAMD_URL_FIND_ALL, NULL, &prefix_added)) {
+
+			query_url = rspamd_mempool_alloc0(task->task_pool,
+											  sizeof(struct rspamd_url));
+			rc = rspamd_url_parse(query_url,
+								  url_str,
+								  strlen(url_str),
+								  task->task_pool,
+								  RSPAMD_URL_PARSE_TEXT);
+
+			if (rc == URI_ERRNO_OK &&
+				url->hostlen > 0) {
+				msg_debug_task("found url %s in query of url"
+							   " %*s",
+							   url_str, url->querylen, rspamd_url_query_unsafe(url));
+
+				if (prefix_added) {
+					query_url->flags |= RSPAMD_URL_FLAG_SCHEMALESS;
+				}
+
+				if (query_url->protocol == PROTOCOL_MAILTO) {
+					if (query_url->userlen == 0) {
+						return TRUE;
+					}
+				}
+
+				rspamd_url_set_add_or_increase(MESSAGE_FIELD(task, urls),
+											   query_url, false);
+			}
+		}
+	}
+
+	return TRUE;
+}
+
+static inline khint_t
+rspamd_url_hash(struct rspamd_url *url)
+{
+	if (url->urllen > 0) {
+		return (khint_t) rspamd_cryptobox_fast_hash(url->string, url->urllen,
+													rspamd_hash_seed());
+	}
+
+	return 0;
+}
+
+static inline khint_t
+rspamd_url_host_hash(struct rspamd_url *url)
+{
+	if (url->hostlen > 0) {
+		return (khint_t) rspamd_cryptobox_fast_hash(rspamd_url_host_unsafe(url),
+													url->hostlen,
+													rspamd_hash_seed());
+	}
+
+	return 0;
+}
+
+/* Compare two emails for building emails tree */
+static inline bool
+rspamd_emails_cmp(struct rspamd_url *u1, struct rspamd_url *u2)
+{
+	gint r;
+
+	if (u1->hostlen != u2->hostlen || u1->hostlen == 0) {
+		return FALSE;
+	}
+	else {
+		if ((r = rspamd_lc_cmp(rspamd_url_host_unsafe(u1),
+							   rspamd_url_host_unsafe(u2), u1->hostlen)) == 0) {
+			if (u1->userlen != u2->userlen || u1->userlen == 0) {
+				return FALSE;
+			}
+			else {
+				return (rspamd_lc_cmp(rspamd_url_user_unsafe(u1),
+									  rspamd_url_user_unsafe(u2),
+									  u1->userlen) == 0);
+			}
+		}
+		else {
+			return r == 0;
+		}
+	}
+
+	return FALSE;
+}
+
+static inline bool
+rspamd_urls_cmp(struct rspamd_url *u1, struct rspamd_url *u2)
+{
+	int r = 0;
+
+	if (u1->protocol != u2->protocol || u1->urllen != u2->urllen) {
+		return false;
+	}
+	else {
+		if (u1->protocol & PROTOCOL_MAILTO) {
+			return rspamd_emails_cmp(u1, u2);
+		}
+
+		r = memcmp(u1->string, u2->string, u1->urllen);
+	}
+
+	return r == 0;
+}
+
+static inline bool
+rspamd_urls_host_cmp(struct rspamd_url *u1, struct rspamd_url *u2)
+{
+	int r = 0;
+
+	if (u1->hostlen != u2->hostlen) {
+		return false;
+	}
+	else {
+		r = memcmp(rspamd_url_host_unsafe(u1), rspamd_url_host_unsafe(u2),
+				   u1->hostlen);
+	}
+
+	return r == 0;
+}
+
+gsize rspamd_url_decode(gchar *dst, const gchar *src, gsize size)
+{
+	gchar *d, ch, c, decoded;
+	const gchar *s;
+	enum {
+		sw_usual = 0,
+		sw_quoted,
+		sw_quoted_second
+	} state;
+
+	d = dst;
+	s = src;
+
+	state = 0;
+	decoded = 0;
+
+	while (size--) {
+
+		ch = *s++;
+
+		switch (state) {
+		case sw_usual:
+
+			if (ch == '%') {
+				state = sw_quoted;
+				break;
+			}
+			else if (ch == '+') {
+				*d++ = ' ';
+			}
+			else {
+				*d++ = ch;
+			}
+			break;
+
+		case sw_quoted:
+
+			if (ch >= '0' && ch <= '9') {
+				decoded = (ch - '0');
+				state = sw_quoted_second;
+				break;
+			}
+
+			c = (ch | 0x20);
+			if (c >= 'a' && c <= 'f') {
+				decoded = (c - 'a' + 10);
+				state = sw_quoted_second;
+				break;
+			}
+
+			/* the invalid quoted character */
+
+			state = sw_usual;
+
+			*d++ = ch;
+
+			break;
+
+		case sw_quoted_second:
+
+			state = sw_usual;
+
+			if (ch >= '0' && ch <= '9') {
+				ch = ((decoded << 4) + ch - '0');
+				*d++ = ch;
+
+				break;
+			}
+
+			c = (u_char) (ch | 0x20);
+			if (c >= 'a' && c <= 'f') {
+				ch = ((decoded << 4) + c - 'a' + 10);
+
+				*d++ = ch;
+				break;
+			}
+
+			/* the invalid quoted character */
+			break;
+		}
+	}
+
+	return (d - dst);
+}
+
+enum rspamd_url_char_class {
+	RSPAMD_URL_UNRESERVED = (1 << 0),
+	RSPAMD_URL_SUBDELIM = (1 << 1),
+	RSPAMD_URL_PATHSAFE = (1 << 2),
+	RSPAMD_URL_QUERYSAFE = (1 << 3),
+	RSPAMD_URL_FRAGMENTSAFE = (1 << 4),
+	RSPAMD_URL_HOSTSAFE = (1 << 5),
+	RSPAMD_URL_USERSAFE = (1 << 6),
+};
+
+#define RSPAMD_URL_FLAGS_HOSTSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_HOSTSAFE | RSPAMD_URL_SUBDELIM)
+#define RSPAMD_URL_FLAGS_USERSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_USERSAFE | RSPAMD_URL_SUBDELIM)
+#define RSPAMD_URL_FLAGS_PATHSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_PATHSAFE | RSPAMD_URL_SUBDELIM)
+#define RSPAMD_URL_FLAGS_QUERYSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_QUERYSAFE | RSPAMD_URL_SUBDELIM)
+#define RSPAMD_URL_FLAGS_FRAGMENTSAFE (RSPAMD_URL_UNRESERVED | RSPAMD_URL_FRAGMENTSAFE | RSPAMD_URL_SUBDELIM)
+
+static const unsigned char rspamd_url_encoding_classes[256] = {
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0 /*   */, RSPAMD_URL_SUBDELIM /* ! */, 0 /* " */, 0 /* # */,
+	RSPAMD_URL_SUBDELIM /* $ */, 0 /* % */, RSPAMD_URL_SUBDELIM /* & */,
+	RSPAMD_URL_SUBDELIM /* ' */, RSPAMD_URL_SUBDELIM /* ( */,
+	RSPAMD_URL_SUBDELIM /* ) */, RSPAMD_URL_SUBDELIM /* * */,
+	RSPAMD_URL_SUBDELIM /* + */, RSPAMD_URL_SUBDELIM /* , */,
+	RSPAMD_URL_UNRESERVED /* - */, RSPAMD_URL_UNRESERVED /* . */,
+	RSPAMD_URL_PATHSAFE | RSPAMD_URL_QUERYSAFE | RSPAMD_URL_FRAGMENTSAFE /* / */,
+	RSPAMD_URL_UNRESERVED /* 0 */, RSPAMD_URL_UNRESERVED /* 1 */,
+	RSPAMD_URL_UNRESERVED /* 2 */, RSPAMD_URL_UNRESERVED /* 3 */,
+	RSPAMD_URL_UNRESERVED /* 4 */, RSPAMD_URL_UNRESERVED /* 5 */,
+	RSPAMD_URL_UNRESERVED /* 6 */, RSPAMD_URL_UNRESERVED /* 7 */,
+	RSPAMD_URL_UNRESERVED /* 8 */, RSPAMD_URL_UNRESERVED /* 9 */,
+	RSPAMD_URL_USERSAFE | RSPAMD_URL_HOSTSAFE | RSPAMD_URL_PATHSAFE | RSPAMD_URL_QUERYSAFE | RSPAMD_URL_FRAGMENTSAFE /* : */,
+	RSPAMD_URL_SUBDELIM /* ; */, 0 /* < */, RSPAMD_URL_SUBDELIM /* = */, 0 /* > */,
+	RSPAMD_URL_QUERYSAFE | RSPAMD_URL_FRAGMENTSAFE /* ? */,
+	RSPAMD_URL_PATHSAFE | RSPAMD_URL_QUERYSAFE | RSPAMD_URL_FRAGMENTSAFE /* @ */,
+	RSPAMD_URL_UNRESERVED /* A */, RSPAMD_URL_UNRESERVED /* B */,
+	RSPAMD_URL_UNRESERVED /* C */, RSPAMD_URL_UNRESERVED /* D */,
+	RSPAMD_URL_UNRESERVED /* E */, RSPAMD_URL_UNRESERVED /* F */,
+	RSPAMD_URL_UNRESERVED /* G */, RSPAMD_URL_UNRESERVED /* H */,
+	RSPAMD_URL_UNRESERVED /* I */, RSPAMD_URL_UNRESERVED /* J */,
+	RSPAMD_URL_UNRESERVED /* K */, RSPAMD_URL_UNRESERVED /* L */,
+	RSPAMD_URL_UNRESERVED /* M */, RSPAMD_URL_UNRESERVED /* N */,
+	RSPAMD_URL_UNRESERVED /* O */, RSPAMD_URL_UNRESERVED /* P */,
+	RSPAMD_URL_UNRESERVED /* Q */, RSPAMD_URL_UNRESERVED /* R */,
+	RSPAMD_URL_UNRESERVED /* S */, RSPAMD_URL_UNRESERVED /* T */,
+	RSPAMD_URL_UNRESERVED /* U */, RSPAMD_URL_UNRESERVED /* V */,
+	RSPAMD_URL_UNRESERVED /* W */, RSPAMD_URL_UNRESERVED /* X */,
+	RSPAMD_URL_UNRESERVED /* Y */, RSPAMD_URL_UNRESERVED /* Z */,
+	RSPAMD_URL_HOSTSAFE /* [ */, 0 /* \ */, RSPAMD_URL_HOSTSAFE /* ] */, 0 /* ^ */,
+	RSPAMD_URL_UNRESERVED /* _ */, 0 /* ` */, RSPAMD_URL_UNRESERVED /* a */,
+	RSPAMD_URL_UNRESERVED /* b */, RSPAMD_URL_UNRESERVED /* c */,
+	RSPAMD_URL_UNRESERVED /* d */, RSPAMD_URL_UNRESERVED /* e */,
+	RSPAMD_URL_UNRESERVED /* f */, RSPAMD_URL_UNRESERVED /* g */,
+	RSPAMD_URL_UNRESERVED /* h */, RSPAMD_URL_UNRESERVED /* i */,
+	RSPAMD_URL_UNRESERVED /* j */, RSPAMD_URL_UNRESERVED /* k */,
+	RSPAMD_URL_UNRESERVED /* l */, RSPAMD_URL_UNRESERVED /* m */,
+	RSPAMD_URL_UNRESERVED /* n */, RSPAMD_URL_UNRESERVED /* o */,
+	RSPAMD_URL_UNRESERVED /* p */, RSPAMD_URL_UNRESERVED /* q */,
+	RSPAMD_URL_UNRESERVED /* r */, RSPAMD_URL_UNRESERVED /* s */,
+	RSPAMD_URL_UNRESERVED /* t */, RSPAMD_URL_UNRESERVED /* u */,
+	RSPAMD_URL_UNRESERVED /* v */, RSPAMD_URL_UNRESERVED /* w */,
+	RSPAMD_URL_UNRESERVED /* x */, RSPAMD_URL_UNRESERVED /* y */,
+	RSPAMD_URL_UNRESERVED /* z */, 0 /* { */, 0 /* | */, 0 /* } */,
+	RSPAMD_URL_UNRESERVED /* ~ */, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+	0, 0, 0, 0, 0, 0, 0, 0, 0};
+
+#define CHECK_URL_COMPONENT(beg, len, flags)                                       \
+	do {                                                                           \
+		for (i = 0; i < (len); i++) {                                              \
+			if ((rspamd_url_encoding_classes[(guchar) (beg)[i]] & (flags)) == 0) { \
+				dlen += 2;                                                         \
+			}                                                                      \
+		}                                                                          \
+	} while (0)
+
+#define ENCODE_URL_COMPONENT(beg, len, flags)                                      \
+	do {                                                                           \
+		for (i = 0; i < (len) && dend > d; i++) {                                  \
+			if ((rspamd_url_encoding_classes[(guchar) (beg)[i]] & (flags)) == 0) { \
+				*d++ = '%';                                                        \
+				*d++ = hexdigests[(guchar) ((beg)[i] >> 4) & 0xf];                 \
+				*d++ = hexdigests[(guchar) (beg)[i] & 0xf];                        \
+			}                                                                      \
+			else {                                                                 \
+				*d++ = (beg)[i];                                                   \
+			}                                                                      \
+		}                                                                          \
+	} while (0)
+
+const gchar *
+rspamd_url_encode(struct rspamd_url *url, gsize *pdlen,
+				  rspamd_mempool_t *pool)
+{
+	guchar *dest, *d, *dend;
+	static const gchar hexdigests[16] = "0123456789ABCDEF";
+	guint i;
+	gsize dlen = 0;
+
+	g_assert(pdlen != NULL && url != NULL && pool != NULL);
+
+	CHECK_URL_COMPONENT(rspamd_url_host_unsafe(url), url->hostlen,
+						RSPAMD_URL_FLAGS_HOSTSAFE);
+	CHECK_URL_COMPONENT(rspamd_url_user_unsafe(url), url->userlen,
+						RSPAMD_URL_FLAGS_USERSAFE);
+	CHECK_URL_COMPONENT(rspamd_url_data_unsafe(url), url->datalen,
+						RSPAMD_URL_FLAGS_PATHSAFE);
+	CHECK_URL_COMPONENT(rspamd_url_query_unsafe(url), url->querylen,
+						RSPAMD_URL_FLAGS_QUERYSAFE);
+	CHECK_URL_COMPONENT(rspamd_url_fragment_unsafe(url), url->fragmentlen,
+						RSPAMD_URL_FLAGS_FRAGMENTSAFE);
+
+	if (dlen == 0) {
+		*pdlen = url->urllen;
+
+		return url->string;
+	}
+
+	/* Need to encode */
+	dlen += url->urllen + sizeof("telephone://"); /* Protocol hack */
+	dest = rspamd_mempool_alloc(pool, dlen + 1);
+	d = dest;
+	dend = d + dlen;
+
+	if (url->protocollen > 0) {
+		if (!(url->protocol & PROTOCOL_UNKNOWN)) {
+			const gchar *known_proto = rspamd_url_protocol_name(url->protocol);
+			d += rspamd_snprintf((gchar *) d, dend - d,
+								 "%s://",
+								 known_proto);
+		}
+		else {
+			d += rspamd_snprintf((gchar *) d, dend - d,
+								 "%*s://",
+								 (gint) url->protocollen, url->string);
+		}
+	}
+	else {
+		d += rspamd_snprintf((gchar *) d, dend - d, "http://");
+	}
+
+	if (url->userlen > 0) {
+		ENCODE_URL_COMPONENT(rspamd_url_user_unsafe(url), url->userlen,
+							 RSPAMD_URL_FLAGS_USERSAFE);
+		*d++ = '@';
+	}
+
+	ENCODE_URL_COMPONENT(rspamd_url_host_unsafe(url), url->hostlen,
+						 RSPAMD_URL_FLAGS_HOSTSAFE);
+
+	if (url->datalen > 0) {
+		*d++ = '/';
+		ENCODE_URL_COMPONENT(rspamd_url_data_unsafe(url), url->datalen,
+							 RSPAMD_URL_FLAGS_PATHSAFE);
+	}
+
+	if (url->querylen > 0) {
+		*d++ = '?';
+		ENCODE_URL_COMPONENT(rspamd_url_query_unsafe(url), url->querylen,
+							 RSPAMD_URL_FLAGS_QUERYSAFE);
+	}
+
+	if (url->fragmentlen > 0) {
+		*d++ = '#';
+		ENCODE_URL_COMPONENT(rspamd_url_fragment_unsafe(url), url->fragmentlen,
+							 RSPAMD_URL_FLAGS_FRAGMENTSAFE);
+	}
+
+	*pdlen = (d - dest);
+
+	return (const gchar *) dest;
+}
+
+gboolean
+rspamd_url_is_domain(int c)
+{
+	return is_domain((guchar) c);
+}
+
+const gchar *
+rspamd_url_protocol_name(enum rspamd_url_protocol proto)
+{
+	const gchar *ret = "unknown";
+
+	switch (proto) {
+	case PROTOCOL_HTTP:
+		ret = "http";
+		break;
+	case PROTOCOL_HTTPS:
+		ret = "https";
+		break;
+	case PROTOCOL_FTP:
+		ret = "ftp";
+		break;
+	case PROTOCOL_FILE:
+		ret = "file";
+		break;
+	case PROTOCOL_MAILTO:
+		ret = "mailto";
+		break;
+	case PROTOCOL_TELEPHONE:
+		ret = "telephone";
+		break;
+	default:
+		break;
+	}
+
+	return ret;
+}
+
+enum rspamd_url_protocol
+rspamd_url_protocol_from_string(const gchar *str)
+{
+	enum rspamd_url_protocol ret = PROTOCOL_UNKNOWN;
+
+	if (strcmp(str, "http") == 0) {
+		ret = PROTOCOL_HTTP;
+	}
+	else if (strcmp(str, "https") == 0) {
+		ret = PROTOCOL_HTTPS;
+	}
+	else if (strcmp(str, "mailto") == 0) {
+		ret = PROTOCOL_MAILTO;
+	}
+	else if (strcmp(str, "ftp") == 0) {
+		ret = PROTOCOL_FTP;
+	}
+	else if (strcmp(str, "file") == 0) {
+		ret = PROTOCOL_FILE;
+	}
+	else if (strcmp(str, "telephone") == 0) {
+		ret = PROTOCOL_TELEPHONE;
+	}
+
+	return ret;
+}
+
+
+bool rspamd_url_set_add_or_increase(khash_t(rspamd_url_hash) * set,
+									struct rspamd_url *u,
+									bool enforce_replace)
+{
+	khiter_t k;
+	gint r;
+
+	k = kh_get(rspamd_url_hash, set, u);
+
+	if (k != kh_end(set)) {
+		/* Existing url */
+		struct rspamd_url *ex = kh_key(set, k);
+#define SUSPICIOUS_URL_FLAGS (RSPAMD_URL_FLAG_PHISHED | RSPAMD_URL_FLAG_OBSCURED | RSPAMD_URL_FLAG_ZW_SPACES)
+		if (enforce_replace) {
+			kh_key(set, k) = u;
+			u->count++;
+		}
+		else {
+			if (u->flags & SUSPICIOUS_URL_FLAGS) {
+				if (!(ex->flags & SUSPICIOUS_URL_FLAGS)) {
+					/* Propagate new url to an old one */
+					kh_key(set, k) = u;
+					u->count++;
+				}
+				else {
+					ex->count++;
+				}
+			}
+			else {
+				ex->count++;
+			}
+		}
+
+		return false;
+	}
+	else {
+		k = kh_put(rspamd_url_hash, set, u, &r);
+	}
+
+	return true;
+}
+
+struct rspamd_url *
+rspamd_url_set_add_or_return(khash_t(rspamd_url_hash) * set,
+							 struct rspamd_url *u)
+{
+	khiter_t k;
+	gint r;
+
+	if (set) {
+		k = kh_get(rspamd_url_hash, set, u);
+
+		if (k != kh_end(set)) {
+			return kh_key(set, k);
+		}
+		else {
+			k = kh_put(rspamd_url_hash, set, u, &r);
+
+			return kh_key(set, k);
+		}
+	}
+
+	return NULL;
+}
+
+bool rspamd_url_host_set_add(khash_t(rspamd_url_host_hash) * set,
+							 struct rspamd_url *u)
+{
+	gint r;
+
+	if (set) {
+		kh_put(rspamd_url_host_hash, set, u, &r);
+
+		if (r == 0) {
+			return false;
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
+bool rspamd_url_set_has(khash_t(rspamd_url_hash) * set, struct rspamd_url *u)
+{
+	khiter_t k;
+
+	if (set) {
+		k = kh_get(rspamd_url_hash, set, u);
+
+		if (k == kh_end(set)) {
+			return false;
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
+bool rspamd_url_host_set_has(khash_t(rspamd_url_host_hash) * set, struct rspamd_url *u)
+{
+	khiter_t k;
+
+	if (set) {
+		k = kh_get(rspamd_url_host_hash, set, u);
+
+		if (k == kh_end(set)) {
+			return false;
+		}
+
+		return true;
+	}
+
+	return false;
+}
+
+bool rspamd_url_flag_from_string(const gchar *str, gint *flag)
+{
+	gint h = rspamd_cryptobox_fast_hash_specific(RSPAMD_CRYPTOBOX_HASHFAST_INDEPENDENT,
+												 str, strlen(str), 0);
+
+	for (int i = 0; i < G_N_ELEMENTS(url_flag_names); i++) {
+		if (url_flag_names[i].hash == h) {
+			*flag |= url_flag_names[i].flag;
+
+			return true;
+		}
+	}
+
+	return false;
+}
+
+
+const gchar *
+rspamd_url_flag_to_string(int flag)
+{
+	for (int i = 0; i < G_N_ELEMENTS(url_flag_names); i++) {
+		if (url_flag_names[i].flag & flag) {
+			return url_flag_names[i].name;
+		}
+	}
+
+	return NULL;
+}
+
+inline int
+rspamd_url_cmp(const struct rspamd_url *u1, const struct rspamd_url *u2)
+{
+	int min_len = MIN(u1->urllen, u2->urllen);
+	int r;
+
+	if (u1->protocol != u2->protocol) {
+		return u1->protocol - u2->protocol;
+	}
+
+	if (u1->protocol & PROTOCOL_MAILTO) {
+		/* Emails specialisation (hosts must be compared in a case insensitive matter */
+		min_len = MIN(u1->hostlen, u2->hostlen);
+
+		if ((r = rspamd_lc_cmp(rspamd_url_host_unsafe(u1),
+							   rspamd_url_host_unsafe(u2), min_len)) == 0) {
+			if (u1->hostlen == u2->hostlen) {
+				if (u1->userlen != u2->userlen || u1->userlen == 0) {
+					r = (int) u1->userlen - (int) u2->userlen;
+				}
+				else {
+					r = memcmp(rspamd_url_user_unsafe(u1),
+							   rspamd_url_user_unsafe(u2),
+							   u1->userlen);
+				}
+			}
+			else {
+				r = u1->hostlen - u2->hostlen;
+			}
+		}
+	}
+	else {
+		if (u1->urllen != u2->urllen) {
+			/* Different length, compare common part and then compare length */
+			r = memcmp(u1->string, u2->string, min_len);
+
+			if (r == 0) {
+				r = u1->urllen - u2->urllen;
+			}
+		}
+		else {
+			/* Equal length */
+			r = memcmp(u1->string, u2->string, u1->urllen);
+		}
+	}
+
+	return r;
+}
+
+int rspamd_url_cmp_qsort(const void *_u1, const void *_u2)
+{
+	const struct rspamd_url *u1 = *(struct rspamd_url **) _u1,
+							*u2 = *(struct rspamd_url **) _u2;
+
+	return rspamd_url_cmp(u1, u2);
+}