Adding upstream version 3.8.1.upstream/3.8.1upstream

Signed-off-by: Daniel Baumann <daniel.baumann@progress-linux.org>

1 files changed, 430 insertions, 0 deletions

diff --git a/src/libserver/url.h b/src/libserver/url.h
new file mode 100644
index 0000000..d1fb8c9
--- /dev/null
+++ b/src/libserver/url.h
@@ -0,0 +1,430 @@
+/* URL check functions */
+#ifndef URL_H
+#define URL_H
+
+#include "config.h"
+#include "mem_pool.h"
+#include "khash.h"
+#include "fstring.h"
+#include "libutil/cxx/utf8_util.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+struct rspamd_task;
+struct rspamd_mime_text_part;
+
+enum rspamd_url_flags {
+	RSPAMD_URL_FLAG_PHISHED = 1u << 0u,
+	RSPAMD_URL_FLAG_NUMERIC = 1u << 1u,
+	RSPAMD_URL_FLAG_OBSCURED = 1u << 2u,
+	RSPAMD_URL_FLAG_REDIRECTED = 1u << 3u,
+	RSPAMD_URL_FLAG_HTML_DISPLAYED = 1u << 4u,
+	RSPAMD_URL_FLAG_FROM_TEXT = 1u << 5u,
+	RSPAMD_URL_FLAG_SUBJECT = 1u << 6u,
+	RSPAMD_URL_FLAG_HOSTENCODED = 1u << 7u,
+	RSPAMD_URL_FLAG_SCHEMAENCODED = 1u << 8u,
+	RSPAMD_URL_FLAG_PATHENCODED = 1u << 9u,
+	RSPAMD_URL_FLAG_QUERYENCODED = 1u << 10u,
+	RSPAMD_URL_FLAG_MISSINGSLASHES = 1u << 11u,
+	RSPAMD_URL_FLAG_IDN = 1u << 12u,
+	RSPAMD_URL_FLAG_HAS_PORT = 1u << 13u,
+	RSPAMD_URL_FLAG_HAS_USER = 1u << 14u,
+	RSPAMD_URL_FLAG_SCHEMALESS = 1u << 15u,
+	RSPAMD_URL_FLAG_UNNORMALISED = 1u << 16u,
+	RSPAMD_URL_FLAG_ZW_SPACES = 1u << 17u,
+	RSPAMD_URL_FLAG_DISPLAY_URL = 1u << 18u,
+	RSPAMD_URL_FLAG_IMAGE = 1u << 19u,
+	RSPAMD_URL_FLAG_QUERY = 1u << 20u,
+	RSPAMD_URL_FLAG_CONTENT = 1u << 21u,
+	RSPAMD_URL_FLAG_NO_TLD = 1u << 22u,
+	RSPAMD_URL_FLAG_TRUNCATED = 1u << 23u,
+	RSPAMD_URL_FLAG_REDIRECT_TARGET = 1u << 24u,
+	RSPAMD_URL_FLAG_INVISIBLE = 1u << 25u,
+	RSPAMD_URL_FLAG_SPECIAL = 1u << 26u,
+
+};
+#define RSPAMD_URL_MAX_FLAG_SHIFT (26u)
+
+struct rspamd_url_tag {
+	const gchar *data;
+	struct rspamd_url_tag *prev, *next;
+};
+
+struct rspamd_url_ext;
+/**
+ * URL structure
+ */
+struct rspamd_url {
+	char *string;
+	char *raw;
+	struct rspamd_url_ext *ext;
+
+	uint32_t flags;
+
+	uint8_t protocol;
+	uint8_t protocollen;
+
+	uint16_t hostshift;
+	uint16_t datashift;
+	uint16_t queryshift;
+	uint16_t fragmentshift;
+	uint16_t tldshift;
+	guint16 usershift;
+	guint16 userlen;
+
+	uint16_t hostlen;
+	uint16_t datalen;
+	uint16_t querylen;
+	uint16_t fragmentlen;
+	uint16_t tldlen;
+	uint16_t count;
+	uint16_t urllen;
+	uint16_t rawlen;
+
+	/* Absolute order of the URL in a message */
+	uint16_t order;
+	/* Order of the URL in a specific part of message */
+	uint16_t part_order;
+};
+
+/**
+ * Rarely used url fields
+ */
+struct rspamd_url_ext {
+	gchar *visible_part;
+	struct rspamd_url *linked_url;
+
+	guint16 port;
+};
+
+#define rspamd_url_user(u) ((u)->userlen > 0 ? (u)->string + (u)->usershift : NULL)
+#define rspamd_url_user_unsafe(u) ((u)->string + (u)->usershift)
+
+#define rspamd_url_host(u) ((u)->hostlen > 0 ? (u)->string + (u)->hostshift : NULL)
+#define rspamd_url_host_unsafe(u) ((u)->string + (u)->hostshift)
+#define rspamd_url_tld_unsafe(u) ((u)->string + (u)->tldshift)
+
+#define rspamd_url_data_unsafe(u) ((u)->string + (u)->datashift)
+#define rspamd_url_query_unsafe(u) ((u)->string + (u)->queryshift)
+#define rspamd_url_fragment_unsafe(u) ((u)->string + (u)->fragmentshift)
+
+enum uri_errno {
+	URI_ERRNO_OK = 0,           /* Parsing went well */
+	URI_ERRNO_EMPTY,            /* The URI string was empty */
+	URI_ERRNO_INVALID_PROTOCOL, /* No protocol was found */
+	URI_ERRNO_INVALID_PORT,     /* Port number is bad */
+	URI_ERRNO_BAD_ENCODING,     /* Bad characters encoding */
+	URI_ERRNO_BAD_FORMAT,
+	URI_ERRNO_TLD_MISSING,
+	URI_ERRNO_HOST_MISSING,
+	URI_ERRNO_TOO_LONG,
+};
+
+enum rspamd_url_protocol {
+	PROTOCOL_FILE = 1u << 0u,
+	PROTOCOL_FTP = 1u << 1u,
+	PROTOCOL_HTTP = 1u << 2u,
+	PROTOCOL_HTTPS = 1u << 3u,
+	PROTOCOL_MAILTO = 1u << 4u,
+	PROTOCOL_TELEPHONE = 1u << 5u,
+	PROTOCOL_UNKNOWN = 1u << 7u,
+};
+
+enum rspamd_url_parse_flags {
+	RSPAMD_URL_PARSE_TEXT = 0u,
+	RSPAMD_URL_PARSE_HREF = (1u << 0u),
+	RSPAMD_URL_PARSE_CHECK = (1u << 1u),
+};
+
+enum rspamd_url_find_type {
+	RSPAMD_URL_FIND_ALL = 0,
+	RSPAMD_URL_FIND_STRICT,
+};
+
+/**
+ * Initialize url library
+ * @param cfg
+ */
+void rspamd_url_init(const gchar *tld_file);
+
+void rspamd_url_deinit(void);
+
+/*
+ * Parse urls inside text
+ * @param pool memory pool
+ * @param task task object
+ * @param part current text part
+ * @param is_html turn on html heuristic
+ */
+void rspamd_url_text_extract(rspamd_mempool_t *pool,
+							 struct rspamd_task *task,
+							 struct rspamd_mime_text_part *part,
+							 uint16_t *cur_order,
+							 enum rspamd_url_find_type how);
+
+/*
+ * Parse a single url into an uri structure
+ * @param pool memory pool
+ * @param uristring text form of url
+ * @param uri url object, must be pre allocated
+ */
+enum uri_errno rspamd_url_parse(struct rspamd_url *uri,
+								gchar *uristring,
+								gsize len,
+								rspamd_mempool_t *pool,
+								enum rspamd_url_parse_flags flags);
+
+/*
+ * Try to extract url from a text
+ * @param pool memory pool
+ * @param begin begin of text
+ * @param len length of text
+ * @param start storage for start position of url found (or NULL)
+ * @param end storage for end position of url found (or NULL)
+ * @param url_str storage for url string(or NULL)
+ * @return TRUE if url is found in specified text
+ */
+gboolean rspamd_url_find(rspamd_mempool_t *pool,
+						 const gchar *begin, gsize len,
+						 gchar **url_str,
+						 enum rspamd_url_find_type how,
+						 goffset *url_pos,
+						 gboolean *prefix_added);
+
+/*
+ * Return text representation of url parsing error
+ */
+const gchar *rspamd_url_strerror(int err);
+
+
+/**
+ * Find TLD for a specified host string
+ * @param in input host
+ * @param inlen length of input
+ * @param out output rspamd_ftok_t with tld position
+ * @return TRUE if tld has been found
+ */
+gboolean rspamd_url_find_tld(const gchar *in, gsize inlen, rspamd_ftok_t *out);
+
+typedef gboolean (*url_insert_function)(struct rspamd_url *url,
+										gsize start_offset, gsize end_offset, void *ud);
+
+/**
+ * Search for multiple urls in text and call `func` for each url found
+ * @param pool
+ * @param in
+ * @param inlen
+ * @param is_html
+ * @param func
+ * @param ud
+ */
+void rspamd_url_find_multiple(rspamd_mempool_t *pool,
+							  const gchar *in, gsize inlen,
+							  enum rspamd_url_find_type how,
+							  GPtrArray *nlines,
+							  url_insert_function func,
+							  gpointer ud);
+
+/**
+ * Search for a single url in text and call `func` for each url found
+ * @param pool
+ * @param in
+ * @param inlen
+ * @param is_html
+ * @param func
+ * @param ud
+ */
+void rspamd_url_find_single(rspamd_mempool_t *pool,
+							const gchar *in, gsize inlen,
+							enum rspamd_url_find_type how,
+							url_insert_function func,
+							gpointer ud);
+
+/**
+ * Generic callback to insert URLs into rspamd_task
+ * @param url
+ * @param start_offset
+ * @param end_offset
+ * @param ud
+ */
+gboolean rspamd_url_task_subject_callback(struct rspamd_url *url,
+										  gsize start_offset,
+										  gsize end_offset, gpointer ud);
+
+/**
+ * Decode URL encoded string in-place and return new length of a string, src and dst are NULL terminated
+ * @param dst
+ * @param src
+ * @param size
+ * @return
+ */
+gsize rspamd_url_decode(gchar *dst, const gchar *src, gsize size);
+
+/**
+ * Encode url if needed. In this case, memory is allocated from the specific pool.
+ * Returns pointer to begin and encoded length in `dlen`
+ * @param url
+ * @param pool
+ * @return
+ */
+const gchar *rspamd_url_encode(struct rspamd_url *url, gsize *dlen,
+							   rspamd_mempool_t *pool);
+
+
+/**
+ * Returns if a character is domain character
+ * @param c
+ * @return
+ */
+gboolean rspamd_url_is_domain(int c);
+
+/**
+ * Returns symbolic name for protocol
+ * @param proto
+ * @return
+ */
+const gchar *rspamd_url_protocol_name(enum rspamd_url_protocol proto);
+
+
+/**
+ * Converts string to a numeric protocol
+ * @param str
+ * @return
+ */
+enum rspamd_url_protocol rspamd_url_protocol_from_string(const gchar *str);
+
+/**
+ * Converts string to a url flag
+ * @param str
+ * @param flag
+ * @return
+ */
+bool rspamd_url_flag_from_string(const gchar *str, gint *flag);
+
+/**
+ * Converts url flag to a string
+ * @param flag
+ * @return
+ */
+const gchar *rspamd_url_flag_to_string(int flag);
+
+/* Defines sets of urls indexed by url as is */
+KHASH_DECLARE(rspamd_url_hash, struct rspamd_url *, char);
+KHASH_DECLARE(rspamd_url_host_hash, struct rspamd_url *, char);
+
+/* Convenience functions for url sets */
+/**
+ * Add an url to set or increase the existing url count
+ * @param set
+ * @param u
+ * @return true if a new url has been added
+ */
+bool rspamd_url_set_add_or_increase(khash_t(rspamd_url_hash) * set,
+									struct rspamd_url *u,
+									bool enforce_replace);
+
+/**
+ * Same as rspamd_url_set_add_or_increase but returns the existing url if found
+ * @param set
+ * @param u
+ * @return
+ */
+struct rspamd_url *rspamd_url_set_add_or_return(khash_t(rspamd_url_hash) * set,
+												struct rspamd_url *u);
+/**
+ * Helper for url host set
+ * @param set
+ * @param u
+ * @return
+ */
+bool rspamd_url_host_set_add(khash_t(rspamd_url_host_hash) * set,
+							 struct rspamd_url *u);
+/**
+ * Checks if a url is in set
+ * @param set
+ * @param u
+ * @return
+ */
+bool rspamd_url_set_has(khash_t(rspamd_url_hash) * set, struct rspamd_url *u);
+
+bool rspamd_url_host_set_has(khash_t(rspamd_url_host_hash) * set, struct rspamd_url *u);
+
+/**
+ * Compares two urls (similar to C comparison functions) lexicographically
+ * @param u1
+ * @param u2
+ * @return
+ */
+int rspamd_url_cmp(const struct rspamd_url *u1, const struct rspamd_url *u2);
+
+/**
+ * Same but used for qsort to sort `struct rspamd_url *[]` array
+ * @param u1
+ * @param u2
+ * @return
+ */
+int rspamd_url_cmp_qsort(const void *u1, const void *u2);
+
+/**
+ * Returns a port for some url
+ * @param u
+ * @return
+ */
+static RSPAMD_PURE_FUNCTION inline uint16_t rspamd_url_get_port(struct rspamd_url *u)
+{
+	if ((u->flags & RSPAMD_URL_FLAG_HAS_PORT) && u->ext) {
+		return u->ext->port;
+	}
+	else {
+		/* Assume standard port */
+		if (u->protocol == PROTOCOL_HTTPS) {
+			return 443;
+		}
+		else {
+			return 80;
+		}
+	}
+}
+
+/**
+ * Returns a port for some url if it is set
+ * @param u
+ * @return
+ */
+static RSPAMD_PURE_FUNCTION inline uint16_t rspamd_url_get_port_if_special(struct rspamd_url *u)
+{
+	if ((u->flags & RSPAMD_URL_FLAG_HAS_PORT) && u->ext) {
+		return u->ext->port;
+	}
+
+	return 0;
+}
+
+/**
+ * Normalize unicode input and set out url flags as appropriate
+ * @param pool
+ * @param input
+ * @param len_out (must be &var)
+ * @param url_flags_out (must be just a var with no dereference)
+ */
+#define rspamd_url_normalise_propagate_flags(pool, input, len_out, url_flags_out) \
+	do {                                                                          \
+		enum rspamd_utf8_normalise_result norm_res;                               \
+		norm_res = rspamd_normalise_unicode_inplace((input), (len_out));          \
+		if (norm_res & RSPAMD_UNICODE_NORM_UNNORMAL) {                            \
+			url_flags_out |= RSPAMD_URL_FLAG_UNNORMALISED;                        \
+		}                                                                         \
+		if (norm_res & RSPAMD_UNICODE_NORM_ZERO_SPACES) {                         \
+			url_flags_out |= RSPAMD_URL_FLAG_ZW_SPACES;                           \
+		}                                                                         \
+		if (norm_res & (RSPAMD_UNICODE_NORM_ERROR)) {                             \
+			url_flags_out |= RSPAMD_URL_FLAG_OBSCURED;                            \
+		}                                                                         \
+	} while (0)
+#ifdef __cplusplus
+}
+#endif
+
+#endif